[llvm-branch-commits] [llvm] [InlineSpiller][AMDGPU] Implement subreg reload during RA spill (PR #175002)
Christudasan Devadasan via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jan 12 05:29:25 PST 2026
https://github.com/cdevadas updated https://github.com/llvm/llvm-project/pull/175002
>From 710afcdc9ea065e1e2642f94f19ee99bf3a58c91 Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Thu, 8 Jan 2026 07:53:18 +0000
Subject: [PATCH 1/3] [InlineSpiller][AMDGPU] Implement subreg reload during RA
spill
Currently, when a virtual register is partially used, the
entire tuple is restored from the spilled location, even if
only a subset of its sub-registers is needed. This patch
introduces support for partial reloads by analyzing actual
register usage and restoring only the required sub-registers.
This improvement enhances register allocation efficiency,
particularly for cases involving tuple virtual registers.
For AMDGPU, this change brings considerable improvements
in workloads that involve matrix operations, large vectors,
and complex control flows.
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 1 +
.../include/llvm/CodeGen/TargetRegisterInfo.h | 10 +
llvm/lib/CodeGen/InlineSpiller.cpp | 57 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 +-
llvm/lib/Target/AMDGPU/#SIRegisterInfo.cpp# | 3938 +++++++++
llvm/lib/Target/AMDGPU/.#SIRegisterInfo.cpp | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 26 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 1 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 14 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 5 +
llvm/lib/Target/ARC/ARCInstrInfo.cpp | 2 +-
llvm/lib/Target/ARC/ARCInstrInfo.h | 1 +
llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp | 2 +-
llvm/lib/Target/ARM/ARMBaseInstrInfo.h | 2 +-
llvm/lib/Target/ARM/Thumb1InstrInfo.cpp | 2 +-
llvm/lib/Target/ARM/Thumb1InstrInfo.h | 2 +-
llvm/lib/Target/ARM/Thumb2InstrInfo.cpp | 2 +-
llvm/lib/Target/ARM/Thumb2InstrInfo.h | 2 +-
llvm/lib/Target/AVR/AVRInstrInfo.cpp | 2 +-
llvm/lib/Target/AVR/AVRInstrInfo.h | 1 +
llvm/lib/Target/BPF/BPFInstrInfo.cpp | 2 +-
llvm/lib/Target/BPF/BPFInstrInfo.h | 2 +-
llvm/lib/Target/CSKY/CSKYInstrInfo.cpp | 2 +-
llvm/lib/Target/CSKY/CSKYInstrInfo.h | 1 +
llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 2 +-
llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 2 +-
llvm/lib/Target/Lanai/LanaiInstrInfo.cpp | 2 +-
llvm/lib/Target/Lanai/LanaiInstrInfo.h | 1 +
.../Target/LoongArch/LoongArchInstrInfo.cpp | 2 +-
.../lib/Target/LoongArch/LoongArchInstrInfo.h | 1 +
llvm/lib/Target/M68k/M68kInstrInfo.cpp | 2 +-
llvm/lib/Target/M68k/M68kInstrInfo.h | 1 +
llvm/lib/Target/MSP430/MSP430InstrInfo.cpp | 2 +-
llvm/lib/Target/MSP430/MSP430InstrInfo.h | 1 +
llvm/lib/Target/Mips/MipsInstrInfo.h | 2 +-
llvm/lib/Target/NVPTX/NVPTXInstrInfo.h | 2 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 2 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.h | 3 +-
llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 6 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 1 +
llvm/lib/Target/Sparc/SparcInstrInfo.cpp | 2 +-
llvm/lib/Target/Sparc/SparcInstrInfo.h | 2 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 2 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.h | 3 +-
llvm/lib/Target/VE/VEInstrInfo.cpp | 2 +-
llvm/lib/Target/VE/VEInstrInfo.h | 3 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 2 +-
llvm/lib/Target/X86/X86InstrInfo.h | 5 +-
llvm/lib/Target/XCore/XCoreInstrInfo.cpp | 2 +-
llvm/lib/Target/XCore/XCoreInstrInfo.h | 5 +-
llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp | 2 +-
llvm/lib/Target/Xtensa/XtensaInstrInfo.h | 2 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 7234 ++++++++---------
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 78 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll | 50 +-
.../dummy-regalloc-priority-advisor.mir | 4 +-
.../AMDGPU/gfx-callable-return-types.ll | 22 +-
.../identical-subrange-spill-infloop.ll | 91 +-
...-reg-class-snippet-copy-use-after-free.mir | 1 +
...nfloop-subrange-spill-inspect-subrange.mir | 5 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 2 +-
llvm/test/CodeGen/AMDGPU/load-global-i16.ll | 6 +-
llvm/test/CodeGen/AMDGPU/load-global-i8.ll | 17 +-
.../ra-inserted-scalar-instructions.mir | 80 +-
...pressure-mitigation-with-subreg-reload.mir | 74 +-
...kip-partial-reload-for-16bit-regaccess.mir | 91 +
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 34 +-
.../CodeGen/AMDGPU/splitkit-copy-bundle.mir | 14 +-
.../AMDGPU/splitkit-copy-live-lanes.mir | 72 +-
.../AMDGPU/splitkit-nolivesubranges.mir | 4 +-
...-last-chance-recoloring-alloc-succeeds.mir | 22 +-
...se-after-free-after-cleanup-failed-vreg.ll | 16 -
74 files changed, 7735 insertions(+), 4330 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/#SIRegisterInfo.cpp#
create mode 120000 llvm/lib/Target/AMDGPU/.#SIRegisterInfo.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
delete mode 100644 llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 11adc190b2a62..045372fad4567 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1215,6 +1215,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
virtual void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const {
llvm_unreachable("Target didn't implement "
"TargetInstrInfo::loadRegFromStackSlot!");
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 5c35cd338feb6..281053cd65922 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1220,6 +1220,11 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return true;
}
+ /// To enable subreg reload of register tuples during RA. This would
+ /// eventually improve the register allocation for the functions that involve
+ /// subreg uses instead of the entire tuple.
+ virtual bool shouldEnableSubRegReload(unsigned SubReg) const { return false; }
+
/// When prioritizing live ranges in register allocation, if this hook returns
/// true then the AllocationPriority of the register class will be treated as
/// more important than whether the range is local to a basic block or global.
@@ -1243,6 +1248,11 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
bool checkAllSuperRegsMarked(const BitVector &RegisterSet,
ArrayRef<MCPhysReg> Exceptions = ArrayRef<MCPhysReg>()) const;
+ virtual const TargetRegisterClass *
+ getConstrainedRegClass(const TargetRegisterClass *RC) const {
+ return RC;
+ }
+
virtual const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 68370303a3aef..c567b88f66a7c 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -217,7 +217,8 @@ class InlineSpiller : public Spiller {
bool coalesceStackAccess(MachineInstr *MI, Register Reg);
bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>,
MachineInstr *LoadMI = nullptr);
- void insertReload(Register VReg, SlotIndex, MachineBasicBlock::iterator MI);
+ void insertReload(Register VReg, unsigned SubReg, SlotIndex,
+ MachineBasicBlock::iterator MI);
void insertSpill(Register VReg, bool isKill, MachineBasicBlock::iterator MI);
void spillAroundUses(Register Reg);
@@ -1112,14 +1113,14 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
return true;
}
-void InlineSpiller::insertReload(Register NewVReg,
+void InlineSpiller::insertReload(Register NewVReg, unsigned SubReg,
SlotIndex Idx,
MachineBasicBlock::iterator MI) {
MachineBasicBlock &MBB = *MI->getParent();
MachineInstrSpan MIS(MI, &MBB);
TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot,
- MRI.getRegClass(NewVReg), Register());
+ MRI.getRegClass(NewVReg), Register(), SubReg);
LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MI);
@@ -1248,10 +1249,51 @@ void InlineSpiller::spillAroundUses(Register Reg) {
// Create a new virtual register for spill/fill.
// FIXME: Infer regclass from instruction alone.
- Register NewVReg = Edit->createFrom(Reg);
+
+ unsigned SubReg = 0;
+ LaneBitmask CoveringLanes = LaneBitmask::getNone();
+ // If the subreg liveness is enabled, identify the subreg use(s) to try
+ // subreg reload. Skip if the instruction also defines the register.
+ // For copy bundles, get the covering lane masks.
+ if (MRI.subRegLivenessEnabled() && !RI.Writes) {
+ for (auto [MI, OpIdx] : Ops) {
+ const MachineOperand &MO = MI->getOperand(OpIdx);
+ assert(MO.isReg() && MO.getReg() == Reg);
+ if (MO.isUse()) {
+ SubReg = MO.getSubReg();
+ if (SubReg)
+ CoveringLanes |= TRI.getSubRegIndexLaneMask(SubReg);
+ }
+ }
+ }
+
+ if (MI.isBundled() && CoveringLanes.any()) {
+ CoveringLanes = LaneBitmask(bit_ceil(CoveringLanes.getAsInteger()) - 1);
+ // Obtain the covering subregister index, including any missing indices
+ // within the identified small range. Although this may be suboptimal due
+ // to gaps in the subregisters that are not part of the copy bundle, it is
+ // benificial when components outside this range of the original tuple can
+ // be completely skipped from the reload.
+ SubReg = TRI.getSubRegIdxFromLaneMask(CoveringLanes);
+ }
+
+ // If the target doesn't support subreg reload, fallback to restoring the
+ // full tuple.
+ if (SubReg && !TRI.shouldEnableSubRegReload(SubReg))
+ SubReg = 0;
+
+ const TargetRegisterClass *OrigRC = MRI.getRegClass(Reg);
+ const TargetRegisterClass *NewRC =
+ SubReg ? TRI.getSubRegisterClass(OrigRC, SubReg) : nullptr;
+
+ // Check if the target needs to constrain the RC further.
+ if (NewRC)
+ NewRC = TRI.getConstrainedRegClass(NewRC);
+
+ Register NewVReg = Edit->createFrom(Reg, NewRC);
if (RI.Reads)
- insertReload(NewVReg, Idx, &MI);
+ insertReload(NewVReg, SubReg, Idx, &MI);
// Rewrite instruction operands.
bool hasLiveDef = false;
@@ -1259,7 +1301,10 @@ void InlineSpiller::spillAroundUses(Register Reg) {
MachineOperand &MO = OpPair.first->getOperand(OpPair.second);
MO.setReg(NewVReg);
if (MO.isUse()) {
- if (!OpPair.first->isRegTiedToDefOperand(OpPair.second))
+ if (SubReg && !MI.isBundled())
+ MO.setSubReg(0);
+ if (!OpPair.first->isRegTiedToDefOperand(OpPair.second) ||
+ (SubReg && !MI.isBundled()))
MO.setIsKill();
} else {
if (!MO.isDead())
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index f04bbf248ca56..f07211325393d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -6157,7 +6157,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 30943533f3667..2ccde3e661de5 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -368,7 +368,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
// This tells target independent code that it is okay to pass instructions
diff --git a/llvm/lib/Target/AMDGPU/#SIRegisterInfo.cpp# b/llvm/lib/Target/AMDGPU/#SIRegisterInfo.cpp#
new file mode 100644
index 0000000000000..3efdad6288814
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/#SIRegisterInfo.cpp#
@@ -0,0 +1,3938 @@
+//===-- SIRegisterInfo.cpp - SI Register Information ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// SI implementation of the TargetRegisterInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPURegisterBankInfo.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUInstPrinter.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+
+using namespace llvm;
+
+#define GET_REGINFO_TARGET_DESC
+#include "AMDGPUGenRegisterInfo.inc"
+
+static cl::opt<bool> EnableSpillSGPRToVGPR(
+ "amdgpu-spill-sgpr-to-vgpr",
+ cl::desc("Enable spilling SGPRs to VGPRs"),
+ cl::ReallyHidden,
+ cl::init(true));
+
+std::array<std::vector<int16_t>, 32> SIRegisterInfo::RegSplitParts;
+std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable;
+
+// Map numbers of DWORDs to indexes in SubRegFromChannelTable.
+// Valid indexes are shifted 1, such that a 0 mapping means unsupported.
+// e.g. for 8 DWORDs (256-bit), SubRegFromChannelTableWidthMap[8] = 8,
+// meaning index 7 in SubRegFromChannelTable.
+static const std::array<unsigned, 17> SubRegFromChannelTableWidthMap = {
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9};
+
+static void emitUnsupportedError(const Function &Fn, const MachineInstr &MI,
+ const Twine &ErrMsg) {
+ Fn.getContext().diagnose(
+ DiagnosticInfoUnsupported(Fn, ErrMsg, MI.getDebugLoc()));
+}
+
+namespace llvm {
+
+// A temporary struct to spill SGPRs.
+// This is mostly to spill SGPRs to memory. Spilling SGPRs into VGPR lanes emits
+// just v_writelane and v_readlane.
+//
+// When spilling to memory, the SGPRs are written into VGPR lanes and the VGPR
+// is saved to scratch (or the other way around for loads).
+// For this, a VGPR is required where the needed lanes can be clobbered. The
+// RegScavenger can provide a VGPR where currently active lanes can be
+// clobbered, but we still need to save inactive lanes.
+// The high-level steps are:
+// - Try to scavenge SGPR(s) to save exec
+// - Try to scavenge VGPR
+// - Save needed, all or inactive lanes of a TmpVGPR
+// - Spill/Restore SGPRs using TmpVGPR
+// - Restore TmpVGPR
+//
+// To save all lanes of TmpVGPR, exec needs to be saved and modified. If we
+// cannot scavenge temporary SGPRs to save exec, we use the following code:
+// buffer_store_dword TmpVGPR ; only if active lanes need to be saved
+// s_not exec, exec
+// buffer_store_dword TmpVGPR ; save inactive lanes
+// s_not exec, exec
+struct SGPRSpillBuilder {
+ struct PerVGPRData {
+ unsigned PerVGPR;
+ unsigned NumVGPRs;
+ int64_t VGPRLanes;
+ };
+
+ // The SGPR to save
+ Register SuperReg;
+ MachineBasicBlock::iterator MI;
+ ArrayRef<int16_t> SplitParts;
+ unsigned NumSubRegs;
+ bool IsKill;
+ const DebugLoc &DL;
+
+ /* When spilling to stack */
+ // The SGPRs are written into this VGPR, which is then written to scratch
+ // (or vice versa for loads).
+ Register TmpVGPR = AMDGPU::NoRegister;
+ // Temporary spill slot to save TmpVGPR to.
+ int TmpVGPRIndex = 0;
+ // If TmpVGPR is live before the spill or if it is scavenged.
+ bool TmpVGPRLive = false;
+ // Scavenged SGPR to save EXEC.
+ Register SavedExecReg = AMDGPU::NoRegister;
+ // Stack index to write the SGPRs to.
+ int Index;
+ unsigned EltSize = 4;
+
+ RegScavenger *RS;
+ MachineBasicBlock *MBB;
+ MachineFunction &MF;
+ SIMachineFunctionInfo &MFI;
+ const SIInstrInfo &TII;
+ const SIRegisterInfo &TRI;
+ bool IsWave32;
+ Register ExecReg;
+ unsigned MovOpc;
+ unsigned NotOpc;
+
+ SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII,
+ bool IsWave32, MachineBasicBlock::iterator MI, int Index,
+ RegScavenger *RS)
+ : SGPRSpillBuilder(TRI, TII, IsWave32, MI, MI->getOperand(0).getReg(),
+ MI->getOperand(0).isKill(), Index, RS) {}
+
+ SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII,
+ bool IsWave32, MachineBasicBlock::iterator MI, Register Reg,
+ bool IsKill, int Index, RegScavenger *RS)
+ : SuperReg(Reg), MI(MI), IsKill(IsKill), DL(MI->getDebugLoc()),
+ Index(Index), RS(RS), MBB(MI->getParent()), MF(*MBB->getParent()),
+ MFI(*MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI),
+ IsWave32(IsWave32) {
+ const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(SuperReg);
+ SplitParts = TRI.getRegSplitParts(RC, EltSize);
+ NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
+
+ if (IsWave32) {
+ ExecReg = AMDGPU::EXEC_LO;
+ MovOpc = AMDGPU::S_MOV_B32;
+ NotOpc = AMDGPU::S_NOT_B32;
+ } else {
+ ExecReg = AMDGPU::EXEC;
+ MovOpc = AMDGPU::S_MOV_B64;
+ NotOpc = AMDGPU::S_NOT_B64;
+ }
+
+ assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
+ assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI &&
+ SuperReg != AMDGPU::EXEC && "exec should never spill");
+ }
+
+ PerVGPRData getPerVGPRData() {
+ PerVGPRData Data;
+ Data.PerVGPR = IsWave32 ? 32 : 64;
+ Data.NumVGPRs = (NumSubRegs + (Data.PerVGPR - 1)) / Data.PerVGPR;
+ Data.VGPRLanes = (1LL << std::min(Data.PerVGPR, NumSubRegs)) - 1LL;
+ return Data;
+ }
+
+ // Tries to scavenge SGPRs to save EXEC and a VGPR. Uses v0 if no VGPR is
+ // free.
+ // Writes these instructions if an SGPR can be scavenged:
+ // s_mov_b64 s[6:7], exec ; Save exec
+ // s_mov_b64 exec, 3 ; Wanted lanemask
+ // buffer_store_dword v1 ; Write scavenged VGPR to emergency slot
+ //
+ // Writes these instructions if no SGPR can be scavenged:
+ // buffer_store_dword v0 ; Only if no free VGPR was found
+ // s_not_b64 exec, exec
+ // buffer_store_dword v0 ; Save inactive lanes
+ // ; exec stays inverted, it is flipped back in
+ // ; restore.
+ void prepare() {
+ // Scavenged temporary VGPR to use. It must be scavenged once for any number
+ // of spilled subregs.
+ // FIXME: The liveness analysis is limited and does not tell if a register
+ // is in use in lanes that are currently inactive. We can never be sure if
+ // a register as actually in use in another lane, so we need to save all
+ // used lanes of the chosen VGPR.
+ assert(RS && "Cannot spill SGPR to memory without RegScavenger");
+ TmpVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false,
+ 0, false);
+
+ // Reserve temporary stack slot
+ TmpVGPRIndex = MFI.getScavengeFI(MF.getFrameInfo(), TRI);
+ if (TmpVGPR) {
+ // Found a register that is dead in the currently active lanes, we only
+ // need to spill inactive lanes.
+ TmpVGPRLive = false;
+ } else {
+ // Pick v0 because it doesn't make a difference.
+ TmpVGPR = AMDGPU::VGPR0;
+ TmpVGPRLive = true;
+ }
+
+ if (TmpVGPRLive) {
+ // We need to inform the scavenger that this index is already in use until
+ // we're done with the custom emergency spill.
+ RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR);
+ }
+
+ // We may end up recursively calling the scavenger, and don't want to re-use
+ // the same register.
+ RS->setRegUsed(TmpVGPR);
+
+ // Try to scavenge SGPRs to save exec
+ assert(!SavedExecReg && "Exec is already saved, refuse to save again");
+ const TargetRegisterClass &RC =
+ IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass;
+ RS->setRegUsed(SuperReg);
+ SavedExecReg = RS->scavengeRegisterBackwards(RC, MI, false, 0, false);
+
+ int64_t VGPRLanes = getPerVGPRData().VGPRLanes;
+
+ if (SavedExecReg) {
+ RS->setRegUsed(SavedExecReg);
+ // Set exec to needed lanes
+ BuildMI(*MBB, MI, DL, TII.get(MovOpc), SavedExecReg).addReg(ExecReg);
+ auto I =
+ BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg).addImm(VGPRLanes);
+ if (!TmpVGPRLive)
+ I.addReg(TmpVGPR, RegState::ImplicitDefine);
+ // Spill needed lanes
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false);
+ } else {
+ // The modify and restore of exec clobber SCC, which we would have to save
+ // and restore. FIXME: We probably would need to reserve a register for
+ // this.
+ if (RS->isRegUsed(AMDGPU::SCC))
+ emitUnsupportedError(MF.getFunction(), *MI,
+ "unhandled SGPR spill to memory");
+
+ // Spill active lanes
+ if (TmpVGPRLive)
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false,
+ /*IsKill*/ false);
+ // Spill inactive lanes
+ auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+ if (!TmpVGPRLive)
+ I.addReg(TmpVGPR, RegState::ImplicitDefine);
+ I->getOperand(2).setIsDead(); // Mark SCC as dead.
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false);
+ }
+ }
+
+ // Writes these instructions if an SGPR can be scavenged:
+ // buffer_load_dword v1 ; Write scavenged VGPR to emergency slot
+ // s_waitcnt vmcnt(0) ; If a free VGPR was found
+ // s_mov_b64 exec, s[6:7] ; Save exec
+ //
+ // Writes these instructions if no SGPR can be scavenged:
+ // buffer_load_dword v0 ; Restore inactive lanes
+ // s_waitcnt vmcnt(0) ; If a free VGPR was found
+ // s_not_b64 exec, exec
+ // buffer_load_dword v0 ; Only if no free VGPR was found
+ void restore() {
+ if (SavedExecReg) {
+ // Restore used lanes
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true,
+ /*IsKill*/ false);
+ // Restore exec
+ auto I = BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg)
+ .addReg(SavedExecReg, RegState::Kill);
+ // Add an implicit use of the load so it is not dead.
+ // FIXME This inserts an unnecessary waitcnt
+ if (!TmpVGPRLive) {
+ I.addReg(TmpVGPR, RegState::ImplicitKill);
+ }
+ } else {
+ // Restore inactive lanes
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true,
+ /*IsKill*/ false);
+ auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+ if (!TmpVGPRLive)
+ I.addReg(TmpVGPR, RegState::ImplicitKill);
+ I->getOperand(2).setIsDead(); // Mark SCC as dead.
+
+ // Restore active lanes
+ if (TmpVGPRLive)
+ TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true);
+ }
+
+ // Inform the scavenger where we're releasing our custom scavenged register.
+ if (TmpVGPRLive) {
+ MachineBasicBlock::iterator RestorePt = std::prev(MI);
+ RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR, &*RestorePt);
+ }
+ }
+
+ // Write TmpVGPR to memory or read TmpVGPR from memory.
+ // Either using a single buffer_load/store if exec is set to the needed mask
+ // or using
+ // buffer_load
+ // s_not exec, exec
+ // buffer_load
+ // s_not exec, exec
+ void readWriteTmpVGPR(unsigned Offset, bool IsLoad) {
+ if (SavedExecReg) {
+ // Spill needed lanes
+ TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad);
+ } else {
+ // The modify and restore of exec clobber SCC, which we would have to save
+ // and restore. FIXME: We probably would need to reserve a register for
+ // this.
+ if (RS->isRegUsed(AMDGPU::SCC))
+ emitUnsupportedError(MF.getFunction(), *MI,
+ "unhandled SGPR spill to memory");
+
+ // Spill active lanes
+ TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad,
+ /*IsKill*/ false);
+ // Spill inactive lanes
+ auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+ Not0->getOperand(2).setIsDead(); // Mark SCC as dead.
+ TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad);
+ auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+ Not1->getOperand(2).setIsDead(); // Mark SCC as dead.
+ }
+ }
+
+ void setMI(MachineBasicBlock *NewMBB, MachineBasicBlock::iterator NewMI) {
+ assert(MBB->getParent() == &MF);
+ MI = NewMI;
+ MBB = NewMBB;
+ }
+};
+
+} // namespace llvm
+
+SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST)
+ : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour(),
+ ST.getAMDGPUDwarfFlavour(),
+ /*PC=*/0,
+ ST.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
+ ST(ST), SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {
+
+ assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
+ getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
+ (getSubRegIndexLaneMask(AMDGPU::lo16) |
+ getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
+ getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() &&
+ "getNumCoveredRegs() will not work with generated subreg masks!");
+
+ RegPressureIgnoredUnits.resize(getNumRegUnits());
+ RegPressureIgnoredUnits.set(
+ static_cast<unsigned>(*regunits(MCRegister::from(AMDGPU::M0)).begin()));
+ for (auto Reg : AMDGPU::VGPR_16RegClass) {
+ if (AMDGPU::isHi16Reg(Reg, *this))
+ RegPressureIgnoredUnits.set(
+ static_cast<unsigned>(*regunits(Reg).begin()));
+ }
+
+ // HACK: Until this is fully tablegen'd.
+ static llvm::once_flag InitializeRegSplitPartsFlag;
+
+ static auto InitializeRegSplitPartsOnce = [this]() {
+ for (unsigned Idx = 1, E = getNumSubRegIndices() - 1; Idx < E; ++Idx) {
+ unsigned Size = getSubRegIdxSize(Idx);
+ if (Size & 15)
+ continue;
+ std::vector<int16_t> &Vec = RegSplitParts[Size / 16 - 1];
+ unsigned Pos = getSubRegIdxOffset(Idx);
+ if (Pos % Size)
+ continue;
+ Pos /= Size;
+ if (Vec.empty()) {
+ unsigned MaxNumParts = 1024 / Size; // Maximum register is 1024 bits.
+ Vec.resize(MaxNumParts);
+ }
+ Vec[Pos] = Idx;
+ }
+ };
+
+ static llvm::once_flag InitializeSubRegFromChannelTableFlag;
+
+ static auto InitializeSubRegFromChannelTableOnce = [this]() {
+ for (auto &Row : SubRegFromChannelTable)
+ Row.fill(AMDGPU::NoSubRegister);
+ for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) {
+ unsigned Width = getSubRegIdxSize(Idx) / 32;
+ unsigned Offset = getSubRegIdxOffset(Idx) / 32;
+ assert(Width < SubRegFromChannelTableWidthMap.size());
+ Width = SubRegFromChannelTableWidthMap[Width];
+ if (Width == 0)
+ continue;
+ unsigned TableIdx = Width - 1;
+ assert(TableIdx < SubRegFromChannelTable.size());
+ assert(Offset < SubRegFromChannelTable[TableIdx].size());
+ SubRegFromChannelTable[TableIdx][Offset] = Idx;
+ }
+ };
+
+ llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce);
+ llvm::call_once(InitializeSubRegFromChannelTableFlag,
+ InitializeSubRegFromChannelTableOnce);
+}
+
+void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved,
+ MCRegister Reg) const {
+ for (MCRegAliasIterator R(Reg, this, true); R.isValid(); ++R)
+ Reserved.set(*R);
+}
+
+// Forced to be here by one .inc
+const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs(
+ const MachineFunction *MF) const {
+ CallingConv::ID CC = MF->getFunction().getCallingConv();
+ switch (CC) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ case CallingConv::Cold:
+ return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_SaveList
+ : CSR_AMDGPU_SaveList;
+ case CallingConv::AMDGPU_Gfx:
+ case CallingConv::AMDGPU_Gfx_WholeWave:
+ return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_SaveList
+ : CSR_AMDGPU_SI_Gfx_SaveList;
+ case CallingConv::AMDGPU_CS_ChainPreserve:
+ return CSR_AMDGPU_CS_ChainPreserve_SaveList;
+ default: {
+ // Dummy to not crash RegisterClassInfo.
+ static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister;
+ return &NoCalleeSavedReg;
+ }
+ }
+}
+
+const MCPhysReg *
+SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const {
+ return nullptr;
+}
+
+const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
+ CallingConv::ID CC) const {
+ switch (CC) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ case CallingConv::Cold:
+ return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_RegMask
+ : CSR_AMDGPU_RegMask;
+ case CallingConv::AMDGPU_Gfx:
+ case CallingConv::AMDGPU_Gfx_WholeWave:
+ return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_RegMask
+ : CSR_AMDGPU_SI_Gfx_RegMask;
+ case CallingConv::AMDGPU_CS_Chain:
+ case CallingConv::AMDGPU_CS_ChainPreserve:
+ // Calls to these functions never return, so we can pretend everything is
+ // preserved.
+ return AMDGPU_AllVGPRs_RegMask;
+ default:
+ return nullptr;
+ }
+}
+
+const uint32_t *SIRegisterInfo::getNoPreservedMask() const {
+ return CSR_AMDGPU_NoRegs_RegMask;
+}
+
+bool SIRegisterInfo::isChainScratchRegister(Register VGPR) {
+ return VGPR >= AMDGPU::VGPR0 && VGPR < AMDGPU::VGPR8;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC,
+ const MachineFunction &MF) const {
+ // FIXME: Should have a helper function like getEquivalentVGPRClass to get the
+ // equivalent AV class. If used one, the verifier will crash after
+ // RegBankSelect in the GISel flow. The aligned regclasses are not fully given
+ // until Instruction selection.
+ if (ST.hasMAIInsts() && (isVGPRClass(RC) || isAGPRClass(RC))) {
+ if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass)
+ return &AMDGPU::AV_32RegClass;
+ if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass)
+ return &AMDGPU::AV_64RegClass;
+ if (RC == &AMDGPU::VReg_64_Align2RegClass ||
+ RC == &AMDGPU::AReg_64_Align2RegClass)
+ return &AMDGPU::AV_64_Align2RegClass;
+ if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass)
+ return &AMDGPU::AV_96RegClass;
+ if (RC == &AMDGPU::VReg_96_Align2RegClass ||
+ RC == &AMDGPU::AReg_96_Align2RegClass)
+ return &AMDGPU::AV_96_Align2RegClass;
+ if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass)
+ return &AMDGPU::AV_128RegClass;
+ if (RC == &AMDGPU::VReg_128_Align2RegClass ||
+ RC == &AMDGPU::AReg_128_Align2RegClass)
+ return &AMDGPU::AV_128_Align2RegClass;
+ if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass)
+ return &AMDGPU::AV_160RegClass;
+ if (RC == &AMDGPU::VReg_160_Align2RegClass ||
+ RC == &AMDGPU::AReg_160_Align2RegClass)
+ return &AMDGPU::AV_160_Align2RegClass;
+ if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass)
+ return &AMDGPU::AV_192RegClass;
+ if (RC == &AMDGPU::VReg_192_Align2RegClass ||
+ RC == &AMDGPU::AReg_192_Align2RegClass)
+ return &AMDGPU::AV_192_Align2RegClass;
+ if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass)
+ return &AMDGPU::AV_256RegClass;
+ if (RC == &AMDGPU::VReg_256_Align2RegClass ||
+ RC == &AMDGPU::AReg_256_Align2RegClass)
+ return &AMDGPU::AV_256_Align2RegClass;
+ if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass)
+ return &AMDGPU::AV_512RegClass;
+ if (RC == &AMDGPU::VReg_512_Align2RegClass ||
+ RC == &AMDGPU::AReg_512_Align2RegClass)
+ return &AMDGPU::AV_512_Align2RegClass;
+ if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass)
+ return &AMDGPU::AV_1024RegClass;
+ if (RC == &AMDGPU::VReg_1024_Align2RegClass ||
+ RC == &AMDGPU::AReg_1024_Align2RegClass)
+ return &AMDGPU::AV_1024_Align2RegClass;
+ }
+
+ return TargetRegisterInfo::getLargestLegalSuperClass(RC, MF);
+}
+
+Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
+ const SIFrameLowering *TFI = ST.getFrameLowering();
+ const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+
+ // During ISel lowering we always reserve the stack pointer in entry and chain
+ // functions, but never actually want to reference it when accessing our own
+ // frame. If we need a frame pointer we use it, but otherwise we can just use
+ // an immediate "0" which we represent by returning NoRegister.
+ if (FuncInfo->isBottomOfStack()) {
+ return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register();
+ }
+ return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg()
+ : FuncInfo->getStackPtrOffsetReg();
+}
+
+bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const {
+ // When we need stack realignment, we can't reference off of the
+ // stack pointer, so we reserve a base pointer.
+ return shouldRealignStack(MF);
+}
+
+Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; }
+
+const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const {
+ return AMDGPU_AllVGPRs_RegMask;
+}
+
+const uint32_t *SIRegisterInfo::getAllAGPRRegMask() const {
+ return AMDGPU_AllAGPRs_RegMask;
+}
+
+const uint32_t *SIRegisterInfo::getAllVectorRegMask() const {
+ return AMDGPU_AllVectorRegs_RegMask;
+}
+
+const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const {
+ return AMDGPU_AllAllocatableSRegs_RegMask;
+}
+
+unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
+ unsigned NumRegs) {
+ assert(NumRegs < SubRegFromChannelTableWidthMap.size());
+ unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs];
+ assert(NumRegIndex && "Not implemented");
+ assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].size());
+ return SubRegFromChannelTable[NumRegIndex - 1][Channel];
+}
+
+MCRegister
+SIRegisterInfo::getAlignedHighSGPRForRC(const MachineFunction &MF,
+ const unsigned Align,
+ const TargetRegisterClass *RC) const {
+ unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), Align) - Align;
+ MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
+ return getMatchingSuperReg(BaseReg, AMDGPU::sub0, RC);
+}
+
+MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg(
+ const MachineFunction &MF) const {
+ return getAlignedHighSGPRForRC(MF, /*Align=*/4, &AMDGPU::SGPR_128RegClass);
+}
+
+BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
+ BitVector Reserved(getNumRegs());
+ Reserved.set(AMDGPU::MODE);
+
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+ // Reserve special purpose registers.
+ //
+ // EXEC_LO and EXEC_HI could be allocated and used as regular register, but
+ // this seems likely to result in bugs, so I'm marking them as reserved.
+ reserveRegisterTuples(Reserved, AMDGPU::EXEC);
+ reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR);
+
+ // M0 has to be reserved so that llvm accepts it as a live-in into a block.
+ reserveRegisterTuples(Reserved, AMDGPU::M0);
+
+ // Reserve src_vccz, src_execz, src_scc.
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC);
+
+ // Reserve the memory aperture registers
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_FLAT_SCRATCH_BASE_LO);
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_FLAT_SCRATCH_BASE_HI);
+
+ // Reserve async counters pseudo registers
+ reserveRegisterTuples(Reserved, AMDGPU::ASYNCcnt);
+ reserveRegisterTuples(Reserved, AMDGPU::TENSORcnt);
+
+ // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen.
+ reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID);
+
+ // Reserve xnack_mask registers - support is not implemented in Codegen.
+ reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK);
+
+ // Reserve lds_direct register - support is not implemented in Codegen.
+ reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT);
+
+ // Reserve Trap Handler registers - support is not implemented in Codegen.
+ reserveRegisterTuples(Reserved, AMDGPU::TBA);
+ reserveRegisterTuples(Reserved, AMDGPU::TMA);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13);
+ reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15);
+
+ // Reserve null register - it shall never be allocated
+ reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL64);
+
+ // Reserve SGPRs.
+ //
+ unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
+ unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
+ for (const TargetRegisterClass *RC : regclasses()) {
+ if (RC->isBaseClass() && isSGPRClass(RC)) {
+ unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
+ for (MCPhysReg Reg : *RC) {
+ unsigned Index = getHWRegIndex(Reg);
+ if (Index + NumRegs > MaxNumSGPRs && Index < TotalNumSGPRs)
+ Reserved.set(Reg);
+ }
+ }
+ }
+
+ Register ScratchRSrcReg = MFI->getScratchRSrcReg();
+ if (ScratchRSrcReg != AMDGPU::NoRegister) {
+ // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we
+ // need to spill.
+ // TODO: May need to reserve a VGPR if doing LDS spilling.
+ reserveRegisterTuples(Reserved, ScratchRSrcReg);
+ }
+
+ Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
+ if (LongBranchReservedReg)
+ reserveRegisterTuples(Reserved, LongBranchReservedReg);
+
+ // We have to assume the SP is needed in case there are calls in the function,
+ // which is detected after the function is lowered. If we aren't really going
+ // to need SP, don't bother reserving it.
+ MCRegister StackPtrReg = MFI->getStackPtrOffsetReg();
+ if (StackPtrReg) {
+ reserveRegisterTuples(Reserved, StackPtrReg);
+ assert(!isSubRegister(ScratchRSrcReg, StackPtrReg));
+ }
+
+ MCRegister FrameReg = MFI->getFrameOffsetReg();
+ if (FrameReg) {
+ reserveRegisterTuples(Reserved, FrameReg);
+ assert(!isSubRegister(ScratchRSrcReg, FrameReg));
+ }
+
+ if (hasBasePointer(MF)) {
+ MCRegister BasePtrReg = getBaseRegister();
+ reserveRegisterTuples(Reserved, BasePtrReg);
+ assert(!isSubRegister(ScratchRSrcReg, BasePtrReg));
+ }
+
+ // FIXME: Use same reserved register introduced in D149775
+ // SGPR used to preserve EXEC MASK around WWM spill/copy instructions.
+ Register ExecCopyReg = MFI->getSGPRForEXECCopy();
+ if (ExecCopyReg)
+ reserveRegisterTuples(Reserved, ExecCopyReg);
+
+ // Reserve VGPRs/AGPRs.
+ //
+ auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction());
+
+ for (const TargetRegisterClass *RC : regclasses()) {
+ if (RC->isBaseClass() && isVGPRClass(RC)) {
+ unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
+ for (MCPhysReg Reg : *RC) {
+ unsigned Index = getHWRegIndex(Reg);
+ if (Index + NumRegs > MaxNumVGPRs)
+ Reserved.set(Reg);
+ }
+ }
+ }
+
+ // Reserve all the AGPRs if there are no instructions to use it.
+ if (!ST.hasMAIInsts())
+ MaxNumAGPRs = 0;
+ for (const TargetRegisterClass *RC : regclasses()) {
+ if (RC->isBaseClass() && isAGPRClass(RC)) {
+ unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
+ for (MCPhysReg Reg : *RC) {
+ unsigned Index = getHWRegIndex(Reg);
+ if (Index + NumRegs > MaxNumAGPRs)
+ Reserved.set(Reg);
+ }
+ }
+ }
+
+ // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
+ // VGPR available at all times.
+ if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
+ reserveRegisterTuples(Reserved, MFI->getVGPRForAGPRCopy());
+ }
+
+ // During wwm-regalloc, reserve the registers for perlane VGPR allocation. The
+ // MFI->getNonWWMRegMask() field will have a valid bitmask only during
+ // wwm-regalloc and it would be empty otherwise.
+ BitVector NonWWMRegMask = MFI->getNonWWMRegMask();
+ if (!NonWWMRegMask.empty()) {
+ for (unsigned RegI = AMDGPU::VGPR0, RegE = AMDGPU::VGPR0 + MaxNumVGPRs;
+ RegI < RegE; ++RegI) {
+ if (NonWWMRegMask.test(RegI))
+ reserveRegisterTuples(Reserved, RegI);
+ }
+ }
+
+ for (Register Reg : MFI->getWWMReservedRegs())
+ reserveRegisterTuples(Reserved, Reg);
+
+ // FIXME: Stop using reserved registers for this.
+ for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs())
+ reserveRegisterTuples(Reserved, Reg);
+
+ for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs())
+ reserveRegisterTuples(Reserved, Reg);
+
+ return Reserved;
+}
+
+bool SIRegisterInfo::isAsmClobberable(const MachineFunction &MF,
+ MCRegister PhysReg) const {
+ return !MF.getRegInfo().isReserved(PhysReg);
+}
+
+bool SIRegisterInfo::shouldRealignStack(const MachineFunction &MF) const {
+ const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
+ // On entry or in chain functions, the base address is 0, so it can't possibly
+ // need any more alignment.
+
+ // FIXME: Should be able to specify the entry frame alignment per calling
+ // convention instead.
+ if (Info->isBottomOfStack())
+ return false;
+
+ return TargetRegisterInfo::shouldRealignStack(MF);
+}
+
+bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const {
+ const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>();
+ if (Info->isEntryFunction()) {
+ const MachineFrameInfo &MFI = Fn.getFrameInfo();
+ return MFI.hasStackObjects() || MFI.hasCalls();
+ }
+
+ // May need scavenger for dealing with callee saved registers.
+ return true;
+}
+
+bool SIRegisterInfo::requiresFrameIndexScavenging(
+ const MachineFunction &MF) const {
+ // Do not use frame virtual registers. They used to be used for SGPRs, but
+ // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the
+ // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a
+ // spill.
+ return false;
+}
+
+bool SIRegisterInfo::requiresFrameIndexReplacementScavenging(
+ const MachineFunction &MF) const {
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+ return MFI.hasStackObjects();
+}
+
+bool SIRegisterInfo::requiresVirtualBaseRegisters(
+ const MachineFunction &) const {
+ // There are no special dedicated stack or frame pointers.
+ return true;
+}
+
+int64_t SIRegisterInfo::getScratchInstrOffset(const MachineInstr *MI) const {
+ assert(SIInstrInfo::isMUBUF(*MI) || SIInstrInfo::isFLATScratch(*MI));
+
+ int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
+ AMDGPU::OpName::offset);
+ return MI->getOperand(OffIdx).getImm();
+}
+
+int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI,
+ int Idx) const {
+ switch (MI->getOpcode()) {
+ case AMDGPU::V_ADD_U32_e32:
+ case AMDGPU::V_ADD_U32_e64:
+ case AMDGPU::V_ADD_CO_U32_e32: {
+ int OtherIdx = Idx == 1 ? 2 : 1;
+ const MachineOperand &OtherOp = MI->getOperand(OtherIdx);
+ return OtherOp.isImm() ? OtherOp.getImm() : 0;
+ }
+ case AMDGPU::V_ADD_CO_U32_e64: {
+ int OtherIdx = Idx == 2 ? 3 : 2;
+ const MachineOperand &OtherOp = MI->getOperand(OtherIdx);
+ return OtherOp.isImm() ? OtherOp.getImm() : 0;
+ }
+ default:
+ break;
+ }
+
+ if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI))
+ return 0;
+
+ assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(),
+ AMDGPU::OpName::vaddr) ||
+ (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(),
+ AMDGPU::OpName::saddr))) &&
+ "Should never see frame index on non-address operand");
+
+ return getScratchInstrOffset(MI);
+}
+
+static bool isFIPlusImmOrVGPR(const SIRegisterInfo &TRI,
+ const MachineInstr &MI) {
+ assert(MI.getDesc().isAdd());
+ const MachineOperand &Src0 = MI.getOperand(1);
+ const MachineOperand &Src1 = MI.getOperand(2);
+
+ if (Src0.isFI()) {
+ return Src1.isImm() || (Src1.isReg() && TRI.isVGPR(MI.getMF()->getRegInfo(),
+ Src1.getReg()));
+ }
+
+ if (Src1.isFI()) {
+ return Src0.isImm() || (Src0.isReg() && TRI.isVGPR(MI.getMF()->getRegInfo(),
+ Src0.getReg()));
+ }
+
+ return false;
+}
+
+bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
+ // TODO: Handle v_add_co_u32, v_or_b32, v_and_b32 and scalar opcodes.
+ switch (MI->getOpcode()) {
+ case AMDGPU::V_ADD_U32_e32: {
+ // TODO: We could handle this but it requires work to avoid violating
+ // operand restrictions.
+ if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e32) < 2 &&
+ !isFIPlusImmOrVGPR(*this, *MI))
+ return false;
+ [[fallthrough]];
+ }
+ case AMDGPU::V_ADD_U32_e64:
+ // FIXME: This optimization is barely profitable enableFlatScratch as-is.
+ //
+ // Much of the benefit with the MUBUF handling is we avoid duplicating the
+ // shift of the frame register, which isn't needed with scratch.
+ //
+ // materializeFrameBaseRegister doesn't know the register classes of the
+ // uses, and unconditionally uses an s_add_i32, which will end up using a
+ // copy for the vector uses.
+ return !ST.enableFlatScratch();
+ case AMDGPU::V_ADD_CO_U32_e32:
+ if (ST.getConstantBusLimit(AMDGPU::V_ADD_CO_U32_e32) < 2 &&
+ !isFIPlusImmOrVGPR(*this, *MI))
+ return false;
+ // We can't deal with the case where the carry out has a use (though this
+ // should never happen)
+ return MI->getOperand(3).isDead();
+ case AMDGPU::V_ADD_CO_U32_e64:
+ // TODO: Should we check use_empty instead?
+ return MI->getOperand(1).isDead();
+ default:
+ break;
+ }
+
+ if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI))
+ return false;
+
+ int64_t FullOffset = Offset + getScratchInstrOffset(MI);
+
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ if (SIInstrInfo::isMUBUF(*MI))
+ return !TII->isLegalMUBUFImmOffset(FullOffset);
+
+ return !TII->isLegalFLATOffset(FullOffset, AMDGPUAS::PRIVATE_ADDRESS,
+ SIInstrFlags::FlatScratch);
+}
+
+Register SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
+ int FrameIdx,
+ int64_t Offset) const {
+ MachineBasicBlock::iterator Ins = MBB->begin();
+ DebugLoc DL; // Defaults to "unknown"
+
+ if (Ins != MBB->end())
+ DL = Ins->getDebugLoc();
+
+ MachineFunction *MF = MBB->getParent();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32
+ : AMDGPU::V_MOV_B32_e32;
+
+ Register BaseReg = MRI.createVirtualRegister(
+ ST.enableFlatScratch() ? &AMDGPU::SReg_32_XEXEC_HIRegClass
+ : &AMDGPU::VGPR_32RegClass);
+
+ if (Offset == 0) {
+ BuildMI(*MBB, Ins, DL, TII->get(MovOpc), BaseReg)
+ .addFrameIndex(FrameIdx);
+ return BaseReg;
+ }
+
+ Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+
+ Register FIReg = MRI.createVirtualRegister(
+ ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass
+ : &AMDGPU::VGPR_32RegClass);
+
+ BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
+ .addImm(Offset);
+ BuildMI(*MBB, Ins, DL, TII->get(MovOpc), FIReg)
+ .addFrameIndex(FrameIdx);
+
+ if (ST.enableFlatScratch() ) {
+ // FIXME: Make sure scc isn't live in.
+ BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_I32), BaseReg)
+ .addReg(OffsetReg, RegState::Kill)
+ .addReg(FIReg)
+ .setOperandDead(3); // scc
+ return BaseReg;
+ }
+
+ TII->getAddNoCarry(*MBB, Ins, DL, BaseReg)
+ .addReg(OffsetReg, RegState::Kill)
+ .addReg(FIReg)
+ .addImm(0); // clamp bit
+
+ return BaseReg;
+}
+
+void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
+ int64_t Offset) const {
+ const SIInstrInfo *TII = ST.getInstrInfo();
+
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_ADD_U32_e32:
+ case AMDGPU::V_ADD_CO_U32_e32: {
+ MachineOperand *FIOp = &MI.getOperand(2);
+ MachineOperand *ImmOp = &MI.getOperand(1);
+ if (!FIOp->isFI())
+ std::swap(FIOp, ImmOp);
+
+ if (!ImmOp->isImm()) {
+ assert(Offset == 0);
+ FIOp->ChangeToRegister(BaseReg, false);
+ TII->legalizeOperandsVOP2(MI.getMF()->getRegInfo(), MI);
+ return;
+ }
+
+ int64_t TotalOffset = ImmOp->getImm() + Offset;
+ if (TotalOffset == 0) {
+ MI.setDesc(TII->get(AMDGPU::COPY));
+ for (unsigned I = MI.getNumOperands() - 1; I != 1; --I)
+ MI.removeOperand(I);
+
+ MI.getOperand(1).ChangeToRegister(BaseReg, false);
+ return;
+ }
+
+ ImmOp->setImm(TotalOffset);
+
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineFunction *MF = MBB->getParent();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+
+ // FIXME: materializeFrameBaseRegister does not know the register class of
+ // the uses of the frame index, and assumes SGPR for enableFlatScratch. Emit
+ // a copy so we have a legal operand and hope the register coalescer can
+ // clean it up.
+ if (isSGPRReg(MRI, BaseReg)) {
+ Register BaseRegVGPR =
+ MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), BaseRegVGPR)
+ .addReg(BaseReg);
+ MI.getOperand(2).ChangeToRegister(BaseRegVGPR, false);
+ } else {
+ MI.getOperand(2).ChangeToRegister(BaseReg, false);
+ }
+ return;
+ }
+ case AMDGPU::V_ADD_U32_e64:
+ case AMDGPU::V_ADD_CO_U32_e64: {
+ int Src0Idx = MI.getNumExplicitDefs();
+ MachineOperand *FIOp = &MI.getOperand(Src0Idx);
+ MachineOperand *ImmOp = &MI.getOperand(Src0Idx + 1);
+ if (!FIOp->isFI())
+ std::swap(FIOp, ImmOp);
+
+ if (!ImmOp->isImm()) {
+ FIOp->ChangeToRegister(BaseReg, false);
+ TII->legalizeOperandsVOP3(MI.getMF()->getRegInfo(), MI);
+ return;
+ }
+
+ int64_t TotalOffset = ImmOp->getImm() + Offset;
+ if (TotalOffset == 0) {
+ MI.setDesc(TII->get(AMDGPU::COPY));
+
+ for (unsigned I = MI.getNumOperands() - 1; I != 1; --I)
+ MI.removeOperand(I);
+
+ MI.getOperand(1).ChangeToRegister(BaseReg, false);
+ } else {
+ FIOp->ChangeToRegister(BaseReg, false);
+ ImmOp->setImm(TotalOffset);
+ }
+
+ return;
+ }
+ default:
+ break;
+ }
+
+ bool IsFlat = TII->isFLATScratch(MI);
+
+#ifndef NDEBUG
+ // FIXME: Is it possible to be storing a frame index to itself?
+ bool SeenFI = false;
+ for (const MachineOperand &MO: MI.operands()) {
+ if (MO.isFI()) {
+ if (SeenFI)
+ llvm_unreachable("should not see multiple frame indices");
+
+ SeenFI = true;
+ }
+ }
+#endif
+
+ MachineOperand *FIOp =
+ TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr
+ : AMDGPU::OpName::vaddr);
+
+ MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset);
+ int64_t NewOffset = OffsetOp->getImm() + Offset;
+
+ assert(FIOp && FIOp->isFI() && "frame index must be address operand");
+ assert(TII->isMUBUF(MI) || TII->isFLATScratch(MI));
+
+ if (IsFlat) {
+ assert(TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS,
+ SIInstrFlags::FlatScratch) &&
+ "offset should be legal");
+ FIOp->ChangeToRegister(BaseReg, false);
+ OffsetOp->setImm(NewOffset);
+ return;
+ }
+
+#ifndef NDEBUG
+ MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
+ assert(SOffset->isImm() && SOffset->getImm() == 0);
+#endif
+
+ assert(TII->isLegalMUBUFImmOffset(NewOffset) && "offset should be legal");
+
+ FIOp->ChangeToRegister(BaseReg, false);
+ OffsetOp->setImm(NewOffset);
+}
+
+bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
+ Register BaseReg,
+ int64_t Offset) const {
+
+ switch (MI->getOpcode()) {
+ case AMDGPU::V_ADD_U32_e32:
+ case AMDGPU::V_ADD_CO_U32_e32:
+ return true;
+ case AMDGPU::V_ADD_U32_e64:
+ case AMDGPU::V_ADD_CO_U32_e64:
+ return ST.hasVOP3Literal() || AMDGPU::isInlinableIntLiteral(Offset);
+ default:
+ break;
+ }
+
+ if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI))
+ return false;
+
+ int64_t NewOffset = Offset + getScratchInstrOffset(MI);
+
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ if (SIInstrInfo::isMUBUF(*MI))
+ return TII->isLegalMUBUFImmOffset(NewOffset);
+
+ return TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS,
+ SIInstrFlags::FlatScratch);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getPointerRegClass(unsigned Kind) const {
+ // This is inaccurate. It depends on the instruction and address space. The
+ // only place where we should hit this is for dealing with frame indexes /
+ // private accesses, so this is correct in that case.
+ return &AMDGPU::VGPR_32RegClass;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
+ return RC == &AMDGPU::SCC_CLASSRegClass ? &AMDGPU::SReg_32RegClass : RC;
+}
+
+static int getOffsetMUBUFStore(unsigned Opc) {
+ switch (Opc) {
+ case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
+ return AMDGPU::BUFFER_STORE_DWORD_OFFSET;
+ case AMDGPU::BUFFER_STORE_BYTE_OFFEN:
+ return AMDGPU::BUFFER_STORE_BYTE_OFFSET;
+ case AMDGPU::BUFFER_STORE_SHORT_OFFEN:
+ return AMDGPU::BUFFER_STORE_SHORT_OFFSET;
+ case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN:
+ return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET;
+ case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN:
+ return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET;
+ case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN:
+ return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET;
+ case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN:
+ return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET;
+ case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN:
+ return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET;
+ default:
+ return -1;
+ }
+}
+
+static int getOffsetMUBUFLoad(unsigned Opc) {
+ switch (Opc) {
+ case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
+ return AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
+ case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN:
+ return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET;
+ case AMDGPU::BUFFER_LOAD_USHORT_OFFEN:
+ return AMDGPU::BUFFER_LOAD_USHORT_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET;
+ case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN:
+ return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET;
+ case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN:
+ return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET;
+ case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN:
+ return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET;
+ case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN:
+ return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET;
+ case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN:
+ return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET;
+ case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN:
+ return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET;
+ default:
+ return -1;
+ }
+}
+
+static int getOffenMUBUFStore(unsigned Opc) {
+ switch (Opc) {
+ case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
+ return AMDGPU::BUFFER_STORE_DWORD_OFFEN;
+ case AMDGPU::BUFFER_STORE_BYTE_OFFSET:
+ return AMDGPU::BUFFER_STORE_BYTE_OFFEN;
+ case AMDGPU::BUFFER_STORE_SHORT_OFFSET:
+ return AMDGPU::BUFFER_STORE_SHORT_OFFEN;
+ case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET:
+ return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN;
+ case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET:
+ return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN;
+ case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET:
+ return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN;
+ case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET:
+ return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN;
+ case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET:
+ return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN;
+ default:
+ return -1;
+ }
+}
+
+static int getOffenMUBUFLoad(unsigned Opc) {
+ switch (Opc) {
+ case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
+ return AMDGPU::BUFFER_LOAD_DWORD_OFFEN;
+ case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET:
+ return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN;
+ case AMDGPU::BUFFER_LOAD_USHORT_OFFSET:
+ return AMDGPU::BUFFER_LOAD_USHORT_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN;
+ case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET:
+ return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN;
+ case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET:
+ return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN;
+ case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET:
+ return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN;
+ case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET:
+ return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN;
+ case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET:
+ return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN;
+ case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET:
+ return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN;
+ default:
+ return -1;
+ }
+}
+
+static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST,
+ MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ int Index, unsigned Lane,
+ unsigned ValueReg, bool IsKill) {
+ MachineFunction *MF = MBB.getParent();
+ SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+
+ MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane);
+
+ if (Reg == AMDGPU::NoRegister)
+ return MachineInstrBuilder();
+
+ bool IsStore = MI->mayStore();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
+
+ unsigned Dst = IsStore ? Reg : ValueReg;
+ unsigned Src = IsStore ? ValueReg : Reg;
+ bool IsVGPR = TRI->isVGPR(MRI, Reg);
+ const DebugLoc &DL = MI->getDebugLoc();
+ if (IsVGPR == TRI->isVGPR(MRI, ValueReg)) {
+ // Spiller during regalloc may restore a spilled register to its superclass.
+ // It could result in AGPR spills restored to VGPRs or the other way around,
+ // making the src and dst with identical regclasses at this point. It just
+ // needs a copy in such cases.
+ auto CopyMIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), Dst)
+ .addReg(Src, getKillRegState(IsKill));
+ CopyMIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+ return CopyMIB;
+ }
+ unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64
+ : AMDGPU::V_ACCVGPR_READ_B32_e64;
+
+ auto MIB = BuildMI(MBB, MI, DL, TII->get(Opc), Dst)
+ .addReg(Src, getKillRegState(IsKill));
+ MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+ return MIB;
+}
+
+// This differs from buildSpillLoadStore by only scavenging a VGPR. It does not
+// need to handle the case where an SGPR may need to be spilled while spilling.
+static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST,
+ MachineFrameInfo &MFI,
+ MachineBasicBlock::iterator MI,
+ int Index,
+ int64_t Offset) {
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ MachineBasicBlock *MBB = MI->getParent();
+ const DebugLoc &DL = MI->getDebugLoc();
+ bool IsStore = MI->mayStore();
+
+ unsigned Opc = MI->getOpcode();
+ int LoadStoreOp = IsStore ?
+ getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc);
+ if (LoadStoreOp == -1)
+ return false;
+
+ const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata);
+ if (spillVGPRtoAGPR(ST, *MBB, MI, Index, 0, Reg->getReg(), false).getInstr())
+ return true;
+
+ MachineInstrBuilder NewMI =
+ BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp))
+ .add(*Reg)
+ .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc))
+ .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset))
+ .addImm(Offset)
+ .addImm(0) // cpol
+ .addImm(0) // swz
+ .cloneMemRefs(*MI);
+
+ const MachineOperand *VDataIn = TII->getNamedOperand(*MI,
+ AMDGPU::OpName::vdata_in);
+ if (VDataIn)
+ NewMI.add(*VDataIn);
+ return true;
+}
+
+static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII,
+ unsigned LoadStoreOp,
+ unsigned EltSize) {
+ bool IsStore = TII->get(LoadStoreOp).mayStore();
+ bool HasVAddr = AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::vaddr);
+ bool UseST =
+ !HasVAddr && !AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::saddr);
+
+ // Handle block load/store first.
+ if (TII->isBlockLoadStore(LoadStoreOp))
+ return LoadStoreOp;
+
+ switch (EltSize) {
+ case 4:
+ LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
+ : AMDGPU::SCRATCH_LOAD_DWORD_SADDR;
+ break;
+ case 8:
+ LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR
+ : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR;
+ break;
+ case 12:
+ LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR
+ : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR;
+ break;
+ case 16:
+ LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR
+ : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR;
+ break;
+ default:
+ llvm_unreachable("Unexpected spill load/store size!");
+ }
+
+ if (HasVAddr)
+ LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp);
+ else if (UseST)
+ LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp);
+
+ return LoadStoreOp;
+}
+
+void SIRegisterInfo::buildSpillLoadStore(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL,
+ unsigned LoadStoreOp, int Index, Register ValueReg, bool IsKill,
+ MCRegister ScratchOffsetReg, int64_t InstOffset, MachineMemOperand *MMO,
+ RegScavenger *RS, LiveRegUnits *LiveUnits) const {
+ assert((!RS || !LiveUnits) && "Only RS or LiveUnits can be set but not both");
+
+ MachineFunction *MF = MBB.getParent();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const MachineFrameInfo &MFI = MF->getFrameInfo();
+ const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
+
+ const MCInstrDesc *Desc = &TII->get(LoadStoreOp);
+ bool IsStore = Desc->mayStore();
+ bool IsFlat = TII->isFLATScratch(LoadStoreOp);
+ bool IsBlock = TII->isBlockLoadStore(LoadStoreOp);
+
+ bool CanClobberSCC = false;
+ bool Scavenged = false;
+ MCRegister SOffset = ScratchOffsetReg;
+
+ const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg);
+ // On gfx90a+ AGPR is a regular VGPR acceptable for loads and stores.
+ const bool IsAGPR = !ST.hasGFX90AInsts() && isAGPRClass(RC);
+ const unsigned RegWidth = AMDGPU::getRegBitWidth(*RC) / 8;
+
+ // Always use 4 byte operations for AGPRs because we need to scavenge
+ // a temporary VGPR.
+ // If we're using a block operation, the element should be the whole block.
+ unsigned EltSize = IsBlock ? RegWidth
+ : (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u)
+ : 4u;
+ unsigned NumSubRegs = RegWidth / EltSize;
+ unsigned Size = NumSubRegs * EltSize;
+ unsigned RemSize = RegWidth - Size;
+ unsigned NumRemSubRegs = RemSize ? 1 : 0;
+ int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
+ int64_t MaterializedOffset = Offset;
+
+ int64_t MaxOffset = Offset + Size + RemSize - EltSize;
+ int64_t ScratchOffsetRegDelta = 0;
+
+ if (IsFlat && EltSize > 4) {
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ Desc = &TII->get(LoadStoreOp);
+ }
+
+ Align Alignment = MFI.getObjectAlign(Index);
+ const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo();
+
+ assert((IsFlat || ((Offset % EltSize) == 0)) &&
+ "unexpected VGPR spill offset");
+
+ // Track a VGPR to use for a constant offset we need to materialize.
+ Register TmpOffsetVGPR;
+
+ // Track a VGPR to use as an intermediate value.
+ Register TmpIntermediateVGPR;
+ bool UseVGPROffset = false;
+
+ // Materialize a VGPR offset required for the given SGPR/VGPR/Immediate
+ // combination.
+ auto MaterializeVOffset = [&](Register SGPRBase, Register TmpVGPR,
+ int64_t VOffset) {
+ // We are using a VGPR offset
+ if (IsFlat && SGPRBase) {
+ // We only have 1 VGPR offset, or 1 SGPR offset. We don't have a free
+ // SGPR, so perform the add as vector.
+ // We don't need a base SGPR in the kernel.
+
+ if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) >= 2) {
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), TmpVGPR)
+ .addReg(SGPRBase)
+ .addImm(VOffset)
+ .addImm(0); // clamp
+ } else {
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
+ .addReg(SGPRBase);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), TmpVGPR)
+ .addImm(VOffset)
+ .addReg(TmpOffsetVGPR);
+ }
+ } else {
+ assert(TmpOffsetVGPR);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
+ .addImm(VOffset);
+ }
+ };
+
+ bool IsOffsetLegal =
+ IsFlat ? TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS,
+ SIInstrFlags::FlatScratch)
+ : TII->isLegalMUBUFImmOffset(MaxOffset);
+ if (!IsOffsetLegal || (IsFlat && !SOffset && !ST.hasFlatScratchSTMode())) {
+ SOffset = MCRegister();
+
+ // We don't have access to the register scavenger if this function is called
+ // during PEI::scavengeFrameVirtualRegs() so use LiveUnits in this case.
+ // TODO: Clobbering SCC is not necessary for scratch instructions in the
+ // entry.
+ if (RS) {
+ SOffset = RS->scavengeRegisterBackwards(AMDGPU::SGPR_32RegClass, MI, false, 0, false);
+
+ // Piggy back on the liveness scan we just did see if SCC is dead.
+ CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC);
+ } else if (LiveUnits) {
+ CanClobberSCC = LiveUnits->available(AMDGPU::SCC);
+ for (MCRegister Reg : AMDGPU::SGPR_32RegClass) {
+ if (LiveUnits->available(Reg) && !MF->getRegInfo().isReserved(Reg)) {
+ SOffset = Reg;
+ break;
+ }
+ }
+ }
+
+ if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC)
+ SOffset = Register();
+
+ if (!SOffset) {
+ UseVGPROffset = true;
+
+ if (RS) {
+ TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0);
+ } else {
+ assert(LiveUnits);
+ for (MCRegister Reg : AMDGPU::VGPR_32RegClass) {
+ if (LiveUnits->available(Reg) && !MF->getRegInfo().isReserved(Reg)) {
+ TmpOffsetVGPR = Reg;
+ break;
+ }
+ }
+ }
+
+ assert(TmpOffsetVGPR);
+ } else if (!SOffset && CanClobberSCC) {
+ // There are no free SGPRs, and since we are in the process of spilling
+ // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true
+ // on SI/CI and on VI it is true until we implement spilling using scalar
+ // stores), we have no way to free up an SGPR. Our solution here is to
+ // add the offset directly to the ScratchOffset or StackPtrOffset
+ // register, and then subtract the offset after the spill to return the
+ // register to it's original value.
+
+ // TODO: If we don't have to do an emergency stack slot spill, converting
+ // to use the VGPR offset is fewer instructions.
+ if (!ScratchOffsetReg)
+ ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg();
+ SOffset = ScratchOffsetReg;
+ ScratchOffsetRegDelta = Offset;
+ } else {
+ Scavenged = true;
+ }
+
+ // We currently only support spilling VGPRs to EltSize boundaries, meaning
+ // we can simplify the adjustment of Offset here to just scale with
+ // WavefrontSize.
+ if (!IsFlat && !UseVGPROffset)
+ Offset *= ST.getWavefrontSize();
+
+ if (!UseVGPROffset && !SOffset)
+ report_fatal_error("could not scavenge SGPR to spill in entry function");
+
+ if (UseVGPROffset) {
+ // We are using a VGPR offset
+ MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, Offset);
+ } else if (ScratchOffsetReg == AMDGPU::NoRegister) {
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset);
+ } else {
+ assert(Offset != 0);
+ auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset)
+ .addReg(ScratchOffsetReg)
+ .addImm(Offset);
+ Add->getOperand(3).setIsDead(); // Mark SCC as dead.
+ }
+
+ Offset = 0;
+ }
+
+ if (IsFlat && SOffset == AMDGPU::NoRegister) {
+ assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0
+ && "Unexpected vaddr for flat scratch with a FI operand");
+
+ if (UseVGPROffset) {
+ LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp);
+ } else {
+ assert(ST.hasFlatScratchSTMode());
+ assert(!TII->isBlockLoadStore(LoadStoreOp) && "Block ops don't have ST");
+ LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp);
+ }
+
+ Desc = &TII->get(LoadStoreOp);
+ }
+
+ for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
+ ++i, RegOffset += EltSize) {
+ if (i == NumSubRegs) {
+ EltSize = RemSize;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ }
+ Desc = &TII->get(LoadStoreOp);
+
+ if (!IsFlat && UseVGPROffset) {
+ int NewLoadStoreOp = IsStore ? getOffenMUBUFStore(LoadStoreOp)
+ : getOffenMUBUFLoad(LoadStoreOp);
+ Desc = &TII->get(NewLoadStoreOp);
+ }
+
+ if (UseVGPROffset && TmpOffsetVGPR == TmpIntermediateVGPR) {
+ // If we are spilling an AGPR beyond the range of the memory instruction
+ // offset and need to use a VGPR offset, we ideally have at least 2
+ // scratch VGPRs. If we don't have a second free VGPR without spilling,
+ // recycle the VGPR used for the offset which requires resetting after
+ // each subregister.
+
+ MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, MaterializedOffset);
+ }
+
+ unsigned NumRegs = EltSize / 4;
+ Register SubReg = e == 1
+ ? ValueReg
+ : Register(getSubReg(ValueReg,
+ getSubRegFromChannel(RegOffset / 4, NumRegs)));
+
+ unsigned SOffsetRegState = 0;
+ unsigned SrcDstRegState = getDefRegState(!IsStore);
+ const bool IsLastSubReg = i + 1 == e;
+ const bool IsFirstSubReg = i == 0;
+ if (IsLastSubReg) {
+ SOffsetRegState |= getKillRegState(Scavenged);
+ // The last implicit use carries the "Kill" flag.
+ SrcDstRegState |= getKillRegState(IsKill);
+ }
+
+ // Make sure the whole register is defined if there are undef components by
+ // adding an implicit def of the super-reg on the first instruction.
+ bool NeedSuperRegDef = e > 1 && IsStore && IsFirstSubReg;
+ bool NeedSuperRegImpOperand = e > 1;
+
+ // Remaining element size to spill into memory after some parts of it
+ // spilled into either AGPRs or VGPRs.
+ unsigned RemEltSize = EltSize;
+
+ // AGPRs to spill VGPRs and vice versa are allocated in a reverse order,
+ // starting from the last lane. In case if a register cannot be completely
+ // spilled into another register that will ensure its alignment does not
+ // change. For targets with VGPR alignment requirement this is important
+ // in case of flat scratch usage as we might get a scratch_load or
+ // scratch_store of an unaligned register otherwise.
+ for (int LaneS = (RegOffset + EltSize) / 4 - 1, Lane = LaneS,
+ LaneE = RegOffset / 4;
+ Lane >= LaneE; --Lane) {
+ bool IsSubReg = e > 1 || EltSize > 4;
+ Register Sub = IsSubReg
+ ? Register(getSubReg(ValueReg, getSubRegFromChannel(Lane)))
+ : ValueReg;
+ auto MIB = spillVGPRtoAGPR(ST, MBB, MI, Index, Lane, Sub, IsKill);
+ if (!MIB.getInstr())
+ break;
+ if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == LaneS && IsFirstSubReg)) {
+ MIB.addReg(ValueReg, RegState::ImplicitDefine);
+ NeedSuperRegDef = false;
+ }
+ if ((IsSubReg || NeedSuperRegImpOperand) && (IsFirstSubReg || IsLastSubReg)) {
+ NeedSuperRegImpOperand = true;
+ unsigned State = SrcDstRegState;
+ if (!IsLastSubReg || (Lane != LaneE))
+ State &= ~RegState::Kill;
+ if (!IsFirstSubReg || (Lane != LaneS))
+ State &= ~RegState::Define;
+ MIB.addReg(ValueReg, RegState::Implicit | State);
+ }
+ RemEltSize -= 4;
+ }
+
+ if (!RemEltSize) // Fully spilled into AGPRs.
+ continue;
+
+ if (RemEltSize != EltSize) { // Partially spilled to AGPRs
+ assert(IsFlat && EltSize > 4);
+
+ unsigned NumRegs = RemEltSize / 4;
+ SubReg = Register(getSubReg(ValueReg,
+ getSubRegFromChannel(RegOffset / 4, NumRegs)));
+ unsigned Opc = getFlatScratchSpillOpcode(TII, LoadStoreOp, RemEltSize);
+ Desc = &TII->get(Opc);
+ }
+
+ unsigned FinalReg = SubReg;
+
+ if (IsAGPR) {
+ assert(EltSize == 4);
+
+ if (!TmpIntermediateVGPR) {
+ TmpIntermediateVGPR = FuncInfo->getVGPRForAGPRCopy();
+ assert(MF->getRegInfo().isReserved(TmpIntermediateVGPR));
+ }
+ if (IsStore) {
+ auto AccRead = BuildMI(MBB, MI, DL,
+ TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64),
+ TmpIntermediateVGPR)
+ .addReg(SubReg, getKillRegState(IsKill));
+ if (NeedSuperRegDef)
+ AccRead.addReg(ValueReg, RegState::ImplicitDefine);
+ if (NeedSuperRegImpOperand && (IsFirstSubReg || IsLastSubReg))
+ AccRead.addReg(ValueReg, RegState::Implicit);
+ AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+ }
+ SubReg = TmpIntermediateVGPR;
+ } else if (UseVGPROffset) {
+ if (!TmpOffsetVGPR) {
+ TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass,
+ MI, false, 0);
+ RS->setRegUsed(TmpOffsetVGPR);
+ }
+ }
+
+ Register FinalValueReg = ValueReg;
+ if (LoadStoreOp == AMDGPU::SCRATCH_LOAD_USHORT_SADDR) {
+ // If we are loading 16-bit value with SRAMECC endabled we need a temp
+ // 32-bit VGPR to load and extract 16-bits into the final register.
+ ValueReg =
+ RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0);
+ SubReg = ValueReg;
+ IsKill = false;
+ }
+
+ MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RegOffset);
+ MachineMemOperand *NewMMO =
+ MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize,
+ commonAlignment(Alignment, RegOffset));
+
+ auto MIB =
+ BuildMI(MBB, MI, DL, *Desc)
+ .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill));
+
+ if (UseVGPROffset) {
+ // For an AGPR spill, we reuse the same temp VGPR for the offset and the
+ // intermediate accvgpr_write.
+ MIB.addReg(TmpOffsetVGPR, getKillRegState(IsLastSubReg && !IsAGPR));
+ }
+
+ if (!IsFlat)
+ MIB.addReg(FuncInfo->getScratchRSrcReg());
+
+ if (SOffset == AMDGPU::NoRegister) {
+ if (!IsFlat) {
+ if (UseVGPROffset && ScratchOffsetReg) {
+ MIB.addReg(ScratchOffsetReg);
+ } else {
+ assert(FuncInfo->isBottomOfStack());
+ MIB.addImm(0);
+ }
+ }
+ } else {
+ MIB.addReg(SOffset, SOffsetRegState);
+ }
+
+ MIB.addImm(Offset + RegOffset);
+
+ bool LastUse = MMO->getFlags() & MOLastUse;
+ MIB.addImm(LastUse ? AMDGPU::CPol::TH_LU : 0); // cpol
+
+ if (!IsFlat)
+ MIB.addImm(0); // swz
+ MIB.addMemOperand(NewMMO);
+
+ if (FinalValueReg != ValueReg) {
+ // Extract 16-bit from the loaded 32-bit value.
+ ValueReg = getSubReg(ValueReg, AMDGPU::lo16);
+ MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B16_t16_e64))
+ .addReg(FinalValueReg, getDefRegState(true))
+ .addImm(0)
+ .addReg(ValueReg, getKillRegState(true))
+ .addImm(0);
+ ValueReg = FinalValueReg;
+ }
+
+ if (!IsAGPR && NeedSuperRegDef)
+ MIB.addReg(ValueReg, RegState::ImplicitDefine);
+
+ if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) {
+ MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64),
+ FinalReg)
+ .addReg(TmpIntermediateVGPR, RegState::Kill);
+ MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+ }
+
+ bool IsSrcDstDef = SrcDstRegState & RegState::Define;
+ bool PartialReloadCopy = (RemEltSize != EltSize) && !IsStore;
+ if (NeedSuperRegImpOperand &&
+ (IsFirstSubReg || (IsLastSubReg && !IsSrcDstDef))) {
+ MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState);
+ if (PartialReloadCopy)
+ MIB.addReg(ValueReg, RegState::Implicit);
+ }
+
+ // The epilog restore of a wwm-scratch register can cause undesired
+ // optimization during machine-cp post PrologEpilogInserter if the same
+ // register was assigned for return value ABI lowering with a COPY
+ // instruction. As given below, with the epilog reload, the earlier COPY
+ // appeared to be dead during machine-cp.
+ // ...
+ // v0 in WWM operation, needs the WWM spill at prolog/epilog.
+ // $vgpr0 = V_WRITELANE_B32 $sgpr20, 0, $vgpr0
+ // ...
+ // Epilog block:
+ // $vgpr0 = COPY $vgpr1 // outgoing value moved to v0
+ // ...
+ // WWM spill restore to preserve the inactive lanes of v0.
+ // $sgpr4_sgpr5 = S_XOR_SAVEEXEC_B64 -1
+ // $vgpr0 = BUFFER_LOAD $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0
+ // $exec = S_MOV_B64 killed $sgpr4_sgpr5
+ // ...
+ // SI_RETURN implicit $vgpr0
+ // ...
+ // To fix it, mark the same reg as a tied op for such restore instructions
+ // so that it marks a usage for the preceding COPY.
+ if (!IsStore && MI != MBB.end() && MI->isReturn() &&
+ MI->readsRegister(SubReg, this)) {
+ MIB.addReg(SubReg, RegState::Implicit);
+ MIB->tieOperands(0, MIB->getNumOperands() - 1);
+ }
+
+ // If we're building a block load, we should add artificial uses for the
+ // CSR VGPRs that are *not* being transferred. This is because liveness
+ // analysis is not aware of the mask, so we need to somehow inform it that
+ // those registers are not available before the load and they should not be
+ // scavenged.
+ if (!IsStore && TII->isBlockLoadStore(LoadStoreOp))
+ addImplicitUsesForBlockCSRLoad(MIB, ValueReg);
+ }
+
+ if (ScratchOffsetRegDelta != 0) {
+ // Subtract the offset we added to the ScratchOffset register.
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset)
+ .addReg(SOffset)
+ .addImm(-ScratchOffsetRegDelta);
+ }
+}
+
+void SIRegisterInfo::addImplicitUsesForBlockCSRLoad(MachineInstrBuilder &MIB,
+ Register BlockReg) const {
+ const MachineFunction *MF = MIB->getMF();
+ const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
+ uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(BlockReg);
+ Register BaseVGPR = getSubReg(BlockReg, AMDGPU::sub0);
+ for (unsigned RegOffset = 1; RegOffset < 32; ++RegOffset)
+ if (!(Mask & (1 << RegOffset)) &&
+ isCalleeSavedPhysReg(BaseVGPR + RegOffset, *MF))
+ MIB.addUse(BaseVGPR + RegOffset, RegState::Implicit);
+}
+
+void SIRegisterInfo::buildVGPRSpillLoadStore(SGPRSpillBuilder &SB, int Index,
+ int Offset, bool IsLoad,
+ bool IsKill) const {
+ // Load/store VGPR
+ MachineFrameInfo &FrameInfo = SB.MF.getFrameInfo();
+ assert(FrameInfo.getStackID(Index) != TargetStackID::SGPRSpill);
+
+ Register FrameReg =
+ FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(SB.MF)
+ ? getBaseRegister()
+ : getFrameRegister(SB.MF);
+
+ Align Alignment = FrameInfo.getObjectAlign(Index);
+ MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SB.MF, Index);
+ MachineMemOperand *MMO = SB.MF.getMachineMemOperand(
+ PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore,
+ SB.EltSize, Alignment);
+
+ if (IsLoad) {
+ unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
+ : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
+ buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, false,
+ FrameReg, (int64_t)Offset * SB.EltSize, MMO, SB.RS);
+ } else {
+ unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
+ : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
+ buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, IsKill,
+ FrameReg, (int64_t)Offset * SB.EltSize, MMO, SB.RS);
+ // This only ever adds one VGPR spill
+ SB.MFI.addToSpilledVGPRs(1);
+ }
+}
+
+bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
+ RegScavenger *RS, SlotIndexes *Indexes,
+ LiveIntervals *LIS, bool OnlyToVGPR,
+ bool SpillToPhysVGPRLane) const {
+ assert(!MI->getOperand(0).isUndef() &&
+ "undef spill should have been deleted earlier");
+
+ SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS);
+
+ ArrayRef<SpilledReg> VGPRSpills =
+ SpillToPhysVGPRLane ? SB.MFI.getSGPRSpillToPhysicalVGPRLanes(Index)
+ : SB.MFI.getSGPRSpillToVirtualVGPRLanes(Index);
+ bool SpillToVGPR = !VGPRSpills.empty();
+ if (OnlyToVGPR && !SpillToVGPR)
+ return false;
+
+ assert(SpillToVGPR || (SB.SuperReg != SB.MFI.getStackPtrOffsetReg() &&
+ SB.SuperReg != SB.MFI.getFrameOffsetReg()));
+
+ if (SpillToVGPR) {
+
+ // Since stack slot coloring pass is trying to optimize SGPR spills,
+ // VGPR lanes (mapped from spill stack slot) may be shared for SGPR
+ // spills of different sizes. This accounts for number of VGPR lanes alloted
+ // equal to the largest SGPR being spilled in them.
+ assert(SB.NumSubRegs <= VGPRSpills.size() &&
+ "Num of SGPRs spilled should be less than or equal to num of "
+ "the VGPR lanes.");
+
+ for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+ SpilledReg Spill = VGPRSpills[i];
+
+ bool IsFirstSubreg = i == 0;
+ bool IsLastSubreg = i == SB.NumSubRegs - 1;
+ bool UseKill = SB.IsKill && IsLastSubreg;
+
+
+ // Mark the "old value of vgpr" input undef only if this is the first sgpr
+ // spill to this specific vgpr in the first basic block.
+ auto MIB = BuildMI(*SB.MBB, MI, SB.DL,
+ SB.TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), Spill.VGPR)
+ .addReg(SubReg, getKillRegState(UseKill))
+ .addImm(Spill.Lane)
+ .addReg(Spill.VGPR);
+ if (Indexes) {
+ if (IsFirstSubreg)
+ Indexes->replaceMachineInstrInMaps(*MI, *MIB);
+ else
+ Indexes->insertMachineInstrInMaps(*MIB);
+ }
+
+ if (IsFirstSubreg && SB.NumSubRegs > 1) {
+ // We may be spilling a super-register which is only partially defined,
+ // and need to ensure later spills think the value is defined.
+ MIB.addReg(SB.SuperReg, RegState::ImplicitDefine);
+ }
+
+ if (SB.NumSubRegs > 1 && (IsFirstSubreg || IsLastSubreg))
+ MIB.addReg(SB.SuperReg, getKillRegState(UseKill) | RegState::Implicit);
+
+ // FIXME: Since this spills to another register instead of an actual
+ // frame index, we should delete the frame index when all references to
+ // it are fixed.
+ }
+ } else {
+ SB.prepare();
+
+ // SubReg carries the "Kill" flag when SubReg == SB.SuperReg.
+ unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill);
+
+ // Per VGPR helper data
+ auto PVD = SB.getPerVGPRData();
+
+ for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) {
+ unsigned TmpVGPRFlags = RegState::Undef;
+
+ // Write sub registers into the VGPR
+ for (unsigned i = Offset * PVD.PerVGPR,
+ e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs);
+ i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+
+ MachineInstrBuilder WriteLane =
+ BuildMI(*SB.MBB, MI, SB.DL,
+ SB.TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), SB.TmpVGPR)
+ .addReg(SubReg, SubKillState)
+ .addImm(i % PVD.PerVGPR)
+ .addReg(SB.TmpVGPR, TmpVGPRFlags);
+ TmpVGPRFlags = 0;
+
+ if (Indexes) {
+ if (i == 0)
+ Indexes->replaceMachineInstrInMaps(*MI, *WriteLane);
+ else
+ Indexes->insertMachineInstrInMaps(*WriteLane);
+ }
+
+ // There could be undef components of a spilled super register.
+ // TODO: Can we detect this and skip the spill?
+ if (SB.NumSubRegs > 1) {
+ // The last implicit use of the SB.SuperReg carries the "Kill" flag.
+ unsigned SuperKillState = 0;
+ if (i + 1 == SB.NumSubRegs)
+ SuperKillState |= getKillRegState(SB.IsKill);
+ WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState);
+ }
+ }
+
+ // Write out VGPR
+ SB.readWriteTmpVGPR(Offset, /*IsLoad*/ false);
+ }
+
+ SB.restore();
+ }
+
+ MI->eraseFromParent();
+ SB.MFI.addToSpilledSGPRs(SB.NumSubRegs);
+
+ if (LIS)
+ LIS->removeAllRegUnitsForPhysReg(SB.SuperReg);
+
+ return true;
+}
+
+bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, int Index,
+ RegScavenger *RS, SlotIndexes *Indexes,
+ LiveIntervals *LIS, bool OnlyToVGPR,
+ bool SpillToPhysVGPRLane) const {
+ SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS);
+
+ ArrayRef<SpilledReg> VGPRSpills =
+ SpillToPhysVGPRLane ? SB.MFI.getSGPRSpillToPhysicalVGPRLanes(Index)
+ : SB.MFI.getSGPRSpillToVirtualVGPRLanes(Index);
+ bool SpillToVGPR = !VGPRSpills.empty();
+ if (OnlyToVGPR && !SpillToVGPR)
+ return false;
+
+ int SpillOffset = MI->getOperand(2).getImm();
+ if (SpillToVGPR) {
+ for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+
+ SpilledReg Spill = VGPRSpills[i + SpillOffset];
+ auto MIB = BuildMI(*SB.MBB, MI, SB.DL,
+ SB.TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg)
+ .addReg(Spill.VGPR)
+ .addImm(Spill.Lane);
+ if (SB.NumSubRegs > 1 && i == 0)
+ MIB.addReg(SB.SuperReg, RegState::ImplicitDefine);
+ if (Indexes) {
+ if (i == e - 1)
+ Indexes->replaceMachineInstrInMaps(*MI, *MIB);
+ else
+ Indexes->insertMachineInstrInMaps(*MIB);
+ }
+ }
+ } else {
+ SB.prepare();
+
+ // Per VGPR helper data
+ auto PVD = SB.getPerVGPRData();
+
+ for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) {
+ // Load in VGPR data
+ SB.readWriteTmpVGPR(Offset, /*IsLoad*/ true);
+
+ // Unpack lanes
+ for (unsigned i = Offset * PVD.PerVGPR,
+ e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs);
+ i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+
+ bool LastSubReg = (i + 1 == e);
+ auto MIB = BuildMI(*SB.MBB, MI, SB.DL,
+ SB.TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg)
+ .addReg(SB.TmpVGPR, getKillRegState(LastSubReg))
+ .addImm(i + SpillOffset);
+ if (SB.NumSubRegs > 1 && i == 0)
+ MIB.addReg(SB.SuperReg, RegState::ImplicitDefine);
+ if (Indexes) {
+ if (i == e - 1)
+ Indexes->replaceMachineInstrInMaps(*MI, *MIB);
+ else
+ Indexes->insertMachineInstrInMaps(*MIB);
+ }
+ }
+ }
+
+ SB.restore();
+ }
+
+ MI->eraseFromParent();
+
+ if (LIS)
+ LIS->removeAllRegUnitsForPhysReg(SB.SuperReg);
+
+ return true;
+}
+
+bool SIRegisterInfo::spillEmergencySGPR(MachineBasicBlock::iterator MI,
+ MachineBasicBlock &RestoreMBB,
+ Register SGPR, RegScavenger *RS) const {
+ SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, SGPR, false, 0,
+ RS);
+ SB.prepare();
+ // Generate the spill of SGPR to SB.TmpVGPR.
+ unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill);
+ auto PVD = SB.getPerVGPRData();
+ for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) {
+ unsigned TmpVGPRFlags = RegState::Undef;
+ // Write sub registers into the VGPR
+ for (unsigned i = Offset * PVD.PerVGPR,
+ e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs);
+ i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+
+ MachineInstrBuilder WriteLane =
+ BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32),
+ SB.TmpVGPR)
+ .addReg(SubReg, SubKillState)
+ .addImm(i % PVD.PerVGPR)
+ .addReg(SB.TmpVGPR, TmpVGPRFlags);
+ TmpVGPRFlags = 0;
+ // There could be undef components of a spilled super register.
+ // TODO: Can we detect this and skip the spill?
+ if (SB.NumSubRegs > 1) {
+ // The last implicit use of the SB.SuperReg carries the "Kill" flag.
+ unsigned SuperKillState = 0;
+ if (i + 1 == SB.NumSubRegs)
+ SuperKillState |= getKillRegState(SB.IsKill);
+ WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState);
+ }
+ }
+ // Don't need to write VGPR out.
+ }
+
+ // Restore clobbered registers in the specified restore block.
+ MI = RestoreMBB.end();
+ SB.setMI(&RestoreMBB, MI);
+ // Generate the restore of SGPR from SB.TmpVGPR.
+ for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) {
+ // Don't need to load VGPR in.
+ // Unpack lanes
+ for (unsigned i = Offset * PVD.PerVGPR,
+ e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs);
+ i < e; ++i) {
+ Register SubReg =
+ SB.NumSubRegs == 1
+ ? SB.SuperReg
+ : Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
+
+ assert(SubReg.isPhysical());
+ bool LastSubReg = (i + 1 == e);
+ auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32),
+ SubReg)
+ .addReg(SB.TmpVGPR, getKillRegState(LastSubReg))
+ .addImm(i);
+ if (SB.NumSubRegs > 1 && i == 0)
+ MIB.addReg(SB.SuperReg, RegState::ImplicitDefine);
+ }
+ }
+ SB.restore();
+
+ SB.MFI.addToSpilledSGPRs(SB.NumSubRegs);
+ return false;
+}
+
+/// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to
+/// a VGPR and the stack slot can be safely eliminated when all other users are
+/// handled.
+bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex(
+ MachineBasicBlock::iterator MI, int FI, RegScavenger *RS,
+ SlotIndexes *Indexes, LiveIntervals *LIS, bool SpillToPhysVGPRLane) const {
+ switch (MI->getOpcode()) {
+ case AMDGPU::SI_SPILL_S1024_SAVE:
+ case AMDGPU::SI_SPILL_S512_SAVE:
+ case AMDGPU::SI_SPILL_S384_SAVE:
+ case AMDGPU::SI_SPILL_S352_SAVE:
+ case AMDGPU::SI_SPILL_S320_SAVE:
+ case AMDGPU::SI_SPILL_S288_SAVE:
+ case AMDGPU::SI_SPILL_S256_SAVE:
+ case AMDGPU::SI_SPILL_S224_SAVE:
+ case AMDGPU::SI_SPILL_S192_SAVE:
+ case AMDGPU::SI_SPILL_S160_SAVE:
+ case AMDGPU::SI_SPILL_S128_SAVE:
+ case AMDGPU::SI_SPILL_S96_SAVE:
+ case AMDGPU::SI_SPILL_S64_SAVE:
+ case AMDGPU::SI_SPILL_S32_SAVE:
+ return spillSGPR(MI, FI, RS, Indexes, LIS, true, SpillToPhysVGPRLane);
+ case AMDGPU::SI_SPILL_S1024_RESTORE:
+ case AMDGPU::SI_SPILL_S512_RESTORE:
+ case AMDGPU::SI_SPILL_S384_RESTORE:
+ case AMDGPU::SI_SPILL_S352_RESTORE:
+ case AMDGPU::SI_SPILL_S320_RESTORE:
+ case AMDGPU::SI_SPILL_S288_RESTORE:
+ case AMDGPU::SI_SPILL_S256_RESTORE:
+ case AMDGPU::SI_SPILL_S224_RESTORE:
+ case AMDGPU::SI_SPILL_S192_RESTORE:
+ case AMDGPU::SI_SPILL_S160_RESTORE:
+ case AMDGPU::SI_SPILL_S128_RESTORE:
+ case AMDGPU::SI_SPILL_S96_RESTORE:
+ case AMDGPU::SI_SPILL_S64_RESTORE:
+ case AMDGPU::SI_SPILL_S32_RESTORE:
+ return restoreSGPR(MI, FI, RS, Indexes, LIS, true, SpillToPhysVGPRLane);
+ default:
+ llvm_unreachable("not an SGPR spill instruction");
+ }
+}
+
+bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
+ int SPAdj, unsigned FIOperandNum,
+ RegScavenger *RS) const {
+ MachineFunction *MF = MI->getMF();
+ MachineBasicBlock *MBB = MI->getParent();
+ SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
+ MachineFrameInfo &FrameInfo = MF->getFrameInfo();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const DebugLoc &DL = MI->getDebugLoc();
+
+ assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?");
+
+ assert(MF->getRegInfo().isReserved(MFI->getScratchRSrcReg()) &&
+ "unreserved scratch RSRC register");
+
+ MachineOperand *FIOp = &MI->getOperand(FIOperandNum);
+ int Index = MI->getOperand(FIOperandNum).getIndex();
+
+ Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF)
+ ? getBaseRegister()
+ : getFrameRegister(*MF);
+
+ switch (MI->getOpcode()) {
+ // SGPR register spill
+ case AMDGPU::SI_SPILL_S1024_SAVE:
+ case AMDGPU::SI_SPILL_S512_SAVE:
+ case AMDGPU::SI_SPILL_S384_SAVE:
+ case AMDGPU::SI_SPILL_S352_SAVE:
+ case AMDGPU::SI_SPILL_S320_SAVE:
+ case AMDGPU::SI_SPILL_S288_SAVE:
+ case AMDGPU::SI_SPILL_S256_SAVE:
+ case AMDGPU::SI_SPILL_S224_SAVE:
+ case AMDGPU::SI_SPILL_S192_SAVE:
+ case AMDGPU::SI_SPILL_S160_SAVE:
+ case AMDGPU::SI_SPILL_S128_SAVE:
+ case AMDGPU::SI_SPILL_S96_SAVE:
+ case AMDGPU::SI_SPILL_S64_SAVE:
+ case AMDGPU::SI_SPILL_S32_SAVE: {
+ return spillSGPR(MI, Index, RS);
+ }
+
+ // SGPR register restore
+ case AMDGPU::SI_SPILL_S1024_RESTORE:
+ case AMDGPU::SI_SPILL_S512_RESTORE:
+ case AMDGPU::SI_SPILL_S384_RESTORE:
+ case AMDGPU::SI_SPILL_S352_RESTORE:
+ case AMDGPU::SI_SPILL_S320_RESTORE:
+ case AMDGPU::SI_SPILL_S288_RESTORE:
+ case AMDGPU::SI_SPILL_S256_RESTORE:
+ case AMDGPU::SI_SPILL_S224_RESTORE:
+ case AMDGPU::SI_SPILL_S192_RESTORE:
+ case AMDGPU::SI_SPILL_S160_RESTORE:
+ case AMDGPU::SI_SPILL_S128_RESTORE:
+ case AMDGPU::SI_SPILL_S96_RESTORE:
+ case AMDGPU::SI_SPILL_S64_RESTORE:
+ case AMDGPU::SI_SPILL_S32_RESTORE: {
+ return restoreSGPR(MI, Index, RS);
+ }
+
+ // VGPR register spill
+ case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE: {
+ // Put mask into M0.
+ BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B32),
+ AMDGPU::M0)
+ .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::mask));
+ [[fallthrough]];
+ }
+ case AMDGPU::SI_SPILL_V1024_SAVE:
+ case AMDGPU::SI_SPILL_V512_SAVE:
+ case AMDGPU::SI_SPILL_V384_SAVE:
+ case AMDGPU::SI_SPILL_V352_SAVE:
+ case AMDGPU::SI_SPILL_V320_SAVE:
+ case AMDGPU::SI_SPILL_V288_SAVE:
+ case AMDGPU::SI_SPILL_V256_SAVE:
+ case AMDGPU::SI_SPILL_V224_SAVE:
+ case AMDGPU::SI_SPILL_V192_SAVE:
+ case AMDGPU::SI_SPILL_V160_SAVE:
+ case AMDGPU::SI_SPILL_V128_SAVE:
+ case AMDGPU::SI_SPILL_V96_SAVE:
+ case AMDGPU::SI_SPILL_V64_SAVE:
+ case AMDGPU::SI_SPILL_V32_SAVE:
+ case AMDGPU::SI_SPILL_V16_SAVE:
+ case AMDGPU::SI_SPILL_A1024_SAVE:
+ case AMDGPU::SI_SPILL_A512_SAVE:
+ case AMDGPU::SI_SPILL_A384_SAVE:
+ case AMDGPU::SI_SPILL_A352_SAVE:
+ case AMDGPU::SI_SPILL_A320_SAVE:
+ case AMDGPU::SI_SPILL_A288_SAVE:
+ case AMDGPU::SI_SPILL_A256_SAVE:
+ case AMDGPU::SI_SPILL_A224_SAVE:
+ case AMDGPU::SI_SPILL_A192_SAVE:
+ case AMDGPU::SI_SPILL_A160_SAVE:
+ case AMDGPU::SI_SPILL_A128_SAVE:
+ case AMDGPU::SI_SPILL_A96_SAVE:
+ case AMDGPU::SI_SPILL_A64_SAVE:
+ case AMDGPU::SI_SPILL_A32_SAVE:
+ case AMDGPU::SI_SPILL_AV1024_SAVE:
+ case AMDGPU::SI_SPILL_AV512_SAVE:
+ case AMDGPU::SI_SPILL_AV384_SAVE:
+ case AMDGPU::SI_SPILL_AV352_SAVE:
+ case AMDGPU::SI_SPILL_AV320_SAVE:
+ case AMDGPU::SI_SPILL_AV288_SAVE:
+ case AMDGPU::SI_SPILL_AV256_SAVE:
+ case AMDGPU::SI_SPILL_AV224_SAVE:
+ case AMDGPU::SI_SPILL_AV192_SAVE:
+ case AMDGPU::SI_SPILL_AV160_SAVE:
+ case AMDGPU::SI_SPILL_AV128_SAVE:
+ case AMDGPU::SI_SPILL_AV96_SAVE:
+ case AMDGPU::SI_SPILL_AV64_SAVE:
+ case AMDGPU::SI_SPILL_AV32_SAVE:
+ case AMDGPU::SI_SPILL_WWM_V32_SAVE:
+ case AMDGPU::SI_SPILL_WWM_AV32_SAVE: {
+ const MachineOperand *VData = TII->getNamedOperand(*MI,
+ AMDGPU::OpName::vdata);
+ if (VData->isUndef()) {
+ MI->eraseFromParent();
+ return true;
+ }
+
+ assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
+ MFI->getStackPtrOffsetReg());
+
+ unsigned Opc;
+ if (MI->getOpcode() == AMDGPU::SI_SPILL_V16_SAVE) {
+ assert(ST.enableFlatScratch() && "Flat Scratch is not enabled!");
+ Opc = AMDGPU::SCRATCH_STORE_SHORT_SADDR_t16;
+ } else {
+ Opc = MI->getOpcode() == AMDGPU::SI_BLOCK_SPILL_V1024_SAVE
+ ? AMDGPU::SCRATCH_STORE_BLOCK_SADDR
+ : ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
+ : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
+ }
+
+ auto *MBB = MI->getParent();
+ bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode());
+ if (IsWWMRegSpill) {
+ TII->insertScratchExecCopy(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy(),
+ RS->isRegUsed(AMDGPU::SCC));
+ }
+ buildSpillLoadStore(
+ *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg,
+ TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
+ *MI->memoperands_begin(), RS);
+ MFI->addToSpilledVGPRs(TII->getNumSubRegsForSpillOp(*MI));
+ if (IsWWMRegSpill)
+ TII->restoreExec(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy());
+
+ MI->eraseFromParent();
+ return true;
+ }
+ case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE: {
+ // Put mask into M0.
+ BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B32),
+ AMDGPU::M0)
+ .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::mask));
+ [[fallthrough]];
+ }
+ case AMDGPU::SI_SPILL_V16_RESTORE:
+ case AMDGPU::SI_SPILL_V32_RESTORE:
+ case AMDGPU::SI_SPILL_V64_RESTORE:
+ case AMDGPU::SI_SPILL_V96_RESTORE:
+ case AMDGPU::SI_SPILL_V128_RESTORE:
+ case AMDGPU::SI_SPILL_V160_RESTORE:
+ case AMDGPU::SI_SPILL_V192_RESTORE:
+ case AMDGPU::SI_SPILL_V224_RESTORE:
+ case AMDGPU::SI_SPILL_V256_RESTORE:
+ case AMDGPU::SI_SPILL_V288_RESTORE:
+ case AMDGPU::SI_SPILL_V320_RESTORE:
+ case AMDGPU::SI_SPILL_V352_RESTORE:
+ case AMDGPU::SI_SPILL_V384_RESTORE:
+ case AMDGPU::SI_SPILL_V512_RESTORE:
+ case AMDGPU::SI_SPILL_V1024_RESTORE:
+ case AMDGPU::SI_SPILL_A32_RESTORE:
+ case AMDGPU::SI_SPILL_A64_RESTORE:
+ case AMDGPU::SI_SPILL_A96_RESTORE:
+ case AMDGPU::SI_SPILL_A128_RESTORE:
+ case AMDGPU::SI_SPILL_A160_RESTORE:
+ case AMDGPU::SI_SPILL_A192_RESTORE:
+ case AMDGPU::SI_SPILL_A224_RESTORE:
+ case AMDGPU::SI_SPILL_A256_RESTORE:
+ case AMDGPU::SI_SPILL_A288_RESTORE:
+ case AMDGPU::SI_SPILL_A320_RESTORE:
+ case AMDGPU::SI_SPILL_A352_RESTORE:
+ case AMDGPU::SI_SPILL_A384_RESTORE:
+ case AMDGPU::SI_SPILL_A512_RESTORE:
+ case AMDGPU::SI_SPILL_A1024_RESTORE:
+ case AMDGPU::SI_SPILL_AV32_RESTORE:
+ case AMDGPU::SI_SPILL_AV64_RESTORE:
+ case AMDGPU::SI_SPILL_AV96_RESTORE:
+ case AMDGPU::SI_SPILL_AV128_RESTORE:
+ case AMDGPU::SI_SPILL_AV160_RESTORE:
+ case AMDGPU::SI_SPILL_AV192_RESTORE:
+ case AMDGPU::SI_SPILL_AV224_RESTORE:
+ case AMDGPU::SI_SPILL_AV256_RESTORE:
+ case AMDGPU::SI_SPILL_AV288_RESTORE:
+ case AMDGPU::SI_SPILL_AV320_RESTORE:
+ case AMDGPU::SI_SPILL_AV352_RESTORE:
+ case AMDGPU::SI_SPILL_AV384_RESTORE:
+ case AMDGPU::SI_SPILL_AV512_RESTORE:
+ case AMDGPU::SI_SPILL_AV1024_RESTORE:
+ case AMDGPU::SI_SPILL_WWM_V32_RESTORE:
+ case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: {
+ const MachineOperand *VData = TII->getNamedOperand(*MI,
+ AMDGPU::OpName::vdata);
+ assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
+ MFI->getStackPtrOffsetReg());
+
+ unsigned Opc;
+ if (MI->getOpcode() == AMDGPU::SI_SPILL_V16_RESTORE) {
+ assert(ST.enableFlatScratch() && "Flat Scratch is not enabled!");
+ Opc = ST.d16PreservesUnusedBits()
+ ? AMDGPU::SCRATCH_LOAD_SHORT_D16_SADDR_t16
+ : AMDGPU::SCRATCH_LOAD_USHORT_SADDR;
+ } else {
+ Opc = MI->getOpcode() == AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE
+ ? AMDGPU::SCRATCH_LOAD_BLOCK_SADDR
+ : ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
+ : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
+ }
+
+ auto *MBB = MI->getParent();
+ bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode());
+ if (IsWWMRegSpill) {
+ TII->insertScratchExecCopy(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy(),
+ RS->isRegUsed(AMDGPU::SCC));
+ }
+
+ buildSpillLoadStore(
+ *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg,
+ TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
+ *MI->memoperands_begin(), RS);
+
+ if (IsWWMRegSpill)
+ TII->restoreExec(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy());
+
+ MI->eraseFromParent();
+ return true;
+ }
+ case AMDGPU::V_ADD_U32_e32:
+ case AMDGPU::V_ADD_U32_e64:
+ case AMDGPU::V_ADD_CO_U32_e32:
+ case AMDGPU::V_ADD_CO_U32_e64: {
+ // TODO: Handle sub, and, or.
+ unsigned NumDefs = MI->getNumExplicitDefs();
+ unsigned Src0Idx = NumDefs;
+
+ bool HasClamp = false;
+ MachineOperand *VCCOp = nullptr;
+
+ switch (MI->getOpcode()) {
+ case AMDGPU::V_ADD_U32_e32:
+ break;
+ case AMDGPU::V_ADD_U32_e64:
+ HasClamp = MI->getOperand(3).getImm();
+ break;
+ case AMDGPU::V_ADD_CO_U32_e32:
+ VCCOp = &MI->getOperand(3);
+ break;
+ case AMDGPU::V_ADD_CO_U32_e64:
+ VCCOp = &MI->getOperand(1);
+ HasClamp = MI->getOperand(4).getImm();
+ break;
+ default:
+ break;
+ }
+ bool DeadVCC = !VCCOp || VCCOp->isDead();
+ MachineOperand &DstOp = MI->getOperand(0);
+ Register DstReg = DstOp.getReg();
+
+ unsigned OtherOpIdx =
+ FIOperandNum == Src0Idx ? FIOperandNum + 1 : Src0Idx;
+ MachineOperand *OtherOp = &MI->getOperand(OtherOpIdx);
+
+ unsigned Src1Idx = Src0Idx + 1;
+ Register MaterializedReg = FrameReg;
+ Register ScavengedVGPR;
+
+ int64_t Offset = FrameInfo.getObjectOffset(Index);
+ // For the non-immediate case, we could fall through to the default
+ // handling, but we do an in-place update of the result register here to
+ // avoid scavenging another register.
+ if (OtherOp->isImm()) {
+ int64_t TotalOffset = OtherOp->getImm() + Offset;
+
+ if (!ST.hasVOP3Literal() && SIInstrInfo::isVOP3(*MI) &&
+ !AMDGPU::isInlinableIntLiteral(TotalOffset)) {
+ // If we can't support a VOP3 literal in the VALU instruction, we
+ // can't specially fold into the add.
+ // TODO: Handle VOP3->VOP2 shrink to support the fold.
+ break;
+ }
+
+ OtherOp->setImm(TotalOffset);
+ Offset = 0;
+ }
+
+ if (FrameReg && !ST.enableFlatScratch()) {
+ // We should just do an in-place update of the result register. However,
+ // the value there may also be used by the add, in which case we need a
+ // temporary register.
+ //
+ // FIXME: The scavenger is not finding the result register in the
+ // common case where the add does not read the register.
+
+ ScavengedVGPR = RS->scavengeRegisterBackwards(
+ AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false, /*SPAdj=*/0);
+
+ // TODO: If we have a free SGPR, it's sometimes better to use a scalar
+ // shift.
+ BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64))
+ .addDef(ScavengedVGPR, RegState::Renamable)
+ .addImm(ST.getWavefrontSizeLog2())
+ .addReg(FrameReg);
+ MaterializedReg = ScavengedVGPR;
+ }
+
+ if ((!OtherOp->isImm() || OtherOp->getImm() != 0) && MaterializedReg) {
+ if (ST.enableFlatScratch() &&
+ !TII->isOperandLegal(*MI, Src1Idx, OtherOp)) {
+ // We didn't need the shift above, so we have an SGPR for the frame
+ // register, but may have a VGPR only operand.
+ //
+ // TODO: On gfx10+, we can easily change the opcode to the e64 version
+ // and use the higher constant bus restriction to avoid this copy.
+
+ if (!ScavengedVGPR) {
+ ScavengedVGPR = RS->scavengeRegisterBackwards(
+ AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false,
+ /*SPAdj=*/0);
+ }
+
+ assert(ScavengedVGPR != DstReg);
+
+ BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), ScavengedVGPR)
+ .addReg(MaterializedReg,
+ MaterializedReg != FrameReg ? RegState::Kill : 0);
+ MaterializedReg = ScavengedVGPR;
+ }
+
+ // TODO: In the flat scratch case, if this is an add of an SGPR, and SCC
+ // is not live, we could use a scalar add + vector add instead of 2
+ // vector adds.
+ auto AddI32 = BuildMI(*MBB, *MI, DL, TII->get(MI->getOpcode()))
+ .addDef(DstReg, RegState::Renamable);
+ if (NumDefs == 2)
+ AddI32.add(MI->getOperand(1));
+
+ unsigned MaterializedRegFlags =
+ MaterializedReg != FrameReg ? RegState::Kill : 0;
+
+ if (isVGPRClass(getPhysRegBaseClass(MaterializedReg))) {
+ // If we know we have a VGPR already, it's more likely the other
+ // operand is a legal vsrc0.
+ AddI32
+ .add(*OtherOp)
+ .addReg(MaterializedReg, MaterializedRegFlags);
+ } else {
+ // Commute operands to avoid violating VOP2 restrictions. This will
+ // typically happen when using scratch.
+ AddI32
+ .addReg(MaterializedReg, MaterializedRegFlags)
+ .add(*OtherOp);
+ }
+
+ if (MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 ||
+ MI->getOpcode() == AMDGPU::V_ADD_U32_e64)
+ AddI32.addImm(0); // clamp
+
+ if (MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e32)
+ AddI32.setOperandDead(3); // Dead vcc
+
+ MaterializedReg = DstReg;
+
+ OtherOp->ChangeToRegister(MaterializedReg, false);
+ OtherOp->setIsKill(true);
+ FIOp->ChangeToImmediate(Offset);
+ Offset = 0;
+ } else if (Offset != 0) {
+ assert(!MaterializedReg);
+ FIOp->ChangeToImmediate(Offset);
+ Offset = 0;
+ } else {
+ if (DeadVCC && !HasClamp) {
+ assert(Offset == 0);
+
+ // TODO: Losing kills and implicit operands. Just mutate to copy and
+ // let lowerCopy deal with it?
+ if (OtherOp->isReg() && OtherOp->getReg() == DstReg) {
+ // Folded to an identity copy.
+ MI->eraseFromParent();
+ return true;
+ }
+
+ // The immediate value should be in OtherOp
+ MI->setDesc(TII->get(AMDGPU::V_MOV_B32_e32));
+ MI->removeOperand(FIOperandNum);
+
+ unsigned NumOps = MI->getNumOperands();
+ for (unsigned I = NumOps - 2; I >= NumDefs + 1; --I)
+ MI->removeOperand(I);
+
+ if (NumDefs == 2)
+ MI->removeOperand(1);
+
+ // The code below can't deal with a mov.
+ return true;
+ }
+
+ // This folded to a constant, but we have to keep the add around for
+ // pointless implicit defs or clamp modifier.
+ FIOp->ChangeToImmediate(0);
+ }
+
+ // Try to improve legality by commuting.
+ if (!TII->isOperandLegal(*MI, Src1Idx) && TII->commuteInstruction(*MI)) {
+ std::swap(FIOp, OtherOp);
+ std::swap(FIOperandNum, OtherOpIdx);
+ }
+
+ // We need at most one mov to satisfy the operand constraints. Prefer to
+ // move the FI operand first, as it may be a literal in a VOP3
+ // instruction.
+ for (unsigned SrcIdx : {FIOperandNum, OtherOpIdx}) {
+ if (!TII->isOperandLegal(*MI, SrcIdx)) {
+ // If commuting didn't make the operands legal, we need to materialize
+ // in a register.
+ // TODO: Can use SGPR on gfx10+ in some cases.
+ if (!ScavengedVGPR) {
+ ScavengedVGPR = RS->scavengeRegisterBackwards(
+ AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false,
+ /*SPAdj=*/0);
+ }
+
+ assert(ScavengedVGPR != DstReg);
+
+ MachineOperand &Src = MI->getOperand(SrcIdx);
+ BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), ScavengedVGPR)
+ .add(Src);
+
+ Src.ChangeToRegister(ScavengedVGPR, false);
+ Src.setIsKill(true);
+ break;
+ }
+ }
+
+ // Fold out add of 0 case that can appear in kernels.
+ if (FIOp->isImm() && FIOp->getImm() == 0 && DeadVCC && !HasClamp) {
+ if (OtherOp->isReg() && OtherOp->getReg() != DstReg) {
+ BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::COPY), DstReg).add(*OtherOp);
+ }
+
+ MI->eraseFromParent();
+ }
+
+ return true;
+ }
+ case AMDGPU::S_ADD_I32:
+ case AMDGPU::S_ADD_U32: {
+ // TODO: Handle s_or_b32, s_and_b32.
+ unsigned OtherOpIdx = FIOperandNum == 1 ? 2 : 1;
+ MachineOperand &OtherOp = MI->getOperand(OtherOpIdx);
+
+ assert(FrameReg || MFI->isBottomOfStack());
+
+ MachineOperand &DstOp = MI->getOperand(0);
+ const DebugLoc &DL = MI->getDebugLoc();
+ Register MaterializedReg = FrameReg;
+
+ // Defend against live scc, which should never happen in practice.
+ bool DeadSCC = MI->getOperand(3).isDead();
+
+ Register TmpReg;
+
+ // FIXME: Scavenger should figure out that the result register is
+ // available. Also should do this for the v_add case.
+ if (OtherOp.isReg() && OtherOp.getReg() != DstOp.getReg())
+ TmpReg = DstOp.getReg();
+
+ if (FrameReg && !ST.enableFlatScratch()) {
+ // FIXME: In the common case where the add does not also read its result
+ // (i.e. this isn't a reg += fi), it's not finding the dest reg as
+ // available.
+ if (!TmpReg)
+ TmpReg = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
+ MI, /*RestoreAfter=*/false, 0,
+ /*AllowSpill=*/false);
+ if (TmpReg) {
+ BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::S_LSHR_B32))
+ .addDef(TmpReg, RegState::Renamable)
+ .addReg(FrameReg)
+ .addImm(ST.getWavefrontSizeLog2())
+ .setOperandDead(3); // Set SCC dead
+ }
+ MaterializedReg = TmpReg;
+ }
+
+ int64_t Offset = FrameInfo.getObjectOffset(Index);
+
+ // For the non-immediate case, we could fall through to the default
+ // handling, but we do an in-place update of the result register here to
+ // avoid scavenging another register.
+ if (OtherOp.isImm()) {
+ OtherOp.setImm(OtherOp.getImm() + Offset);
+ Offset = 0;
+
+ if (MaterializedReg)
+ FIOp->ChangeToRegister(MaterializedReg, false);
+ else
+ FIOp->ChangeToImmediate(0);
+ } else if (MaterializedReg) {
+ // If we can't fold the other operand, do another increment.
+ Register DstReg = DstOp.getReg();
+
+ if (!TmpReg && MaterializedReg == FrameReg) {
+ TmpReg = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
+ MI, /*RestoreAfter=*/false, 0,
+ /*AllowSpill=*/false);
+ DstReg = TmpReg;
+ }
+
+ if (TmpReg) {
+ auto AddI32 = BuildMI(*MBB, *MI, DL, MI->getDesc())
+ .addDef(DstReg, RegState::Renamable)
+ .addReg(MaterializedReg, RegState::Kill)
+ .add(OtherOp);
+ if (DeadSCC)
+ AddI32.setOperandDead(3);
+
+ MaterializedReg = DstReg;
+
+ OtherOp.ChangeToRegister(MaterializedReg, false);
+ OtherOp.setIsKill(true);
+ OtherOp.setIsRenamable(true);
+ }
+ FIOp->ChangeToImmediate(Offset);
+ } else {
+ // If we don't have any other offset to apply, we can just directly
+ // interpret the frame index as the offset.
+ FIOp->ChangeToImmediate(Offset);
+ }
+
+ if (DeadSCC && OtherOp.isImm() && OtherOp.getImm() == 0) {
+ assert(Offset == 0);
+ MI->removeOperand(3);
+ MI->removeOperand(OtherOpIdx);
+ MI->setDesc(TII->get(FIOp->isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
+ } else if (DeadSCC && FIOp->isImm() && FIOp->getImm() == 0) {
+ assert(Offset == 0);
+ MI->removeOperand(3);
+ MI->removeOperand(FIOperandNum);
+ MI->setDesc(
+ TII->get(OtherOp.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
+ }
+
+ assert(!FIOp->isFI());
+ return true;
+ }
+ default: {
+ break;
+ }
+ }
+
+ int64_t Offset = FrameInfo.getObjectOffset(Index);
+ if (ST.enableFlatScratch()) {
+ if (TII->isFLATScratch(*MI)) {
+ assert(
+ (int16_t)FIOperandNum ==
+ AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::saddr));
+
+ // The offset is always swizzled, just replace it
+ if (FrameReg)
+ FIOp->ChangeToRegister(FrameReg, false);
+
+ MachineOperand *OffsetOp =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
+ int64_t NewOffset = Offset + OffsetOp->getImm();
+ if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS,
+ SIInstrFlags::FlatScratch)) {
+ OffsetOp->setImm(NewOffset);
+ if (FrameReg)
+ return false;
+ Offset = 0;
+ }
+
+ if (!Offset) {
+ unsigned Opc = MI->getOpcode();
+ int NewOpc = -1;
+ if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr)) {
+ NewOpc = AMDGPU::getFlatScratchInstSVfromSVS(Opc);
+ } else if (ST.hasFlatScratchSTMode()) {
+ // On GFX10 we have ST mode to use no registers for an address.
+ // Otherwise we need to materialize 0 into an SGPR.
+ NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc);
+ }
+
+ if (NewOpc != -1) {
+ // removeOperand doesn't fixup tied operand indexes as it goes, so
+ // it asserts. Untie vdst_in for now and retie them afterwards.
+ int VDstIn =
+ AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);
+ bool TiedVDst = VDstIn != -1 && MI->getOperand(VDstIn).isReg() &&
+ MI->getOperand(VDstIn).isTied();
+ if (TiedVDst)
+ MI->untieRegOperand(VDstIn);
+
+ MI->removeOperand(
+ AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr));
+
+ if (TiedVDst) {
+ int NewVDst =
+ AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
+ int NewVDstIn =
+ AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst_in);
+ assert(NewVDst != -1 && NewVDstIn != -1 && "Must be tied!");
+ MI->tieOperands(NewVDst, NewVDstIn);
+ }
+ MI->setDesc(TII->get(NewOpc));
+ return false;
+ }
+ }
+ }
+
+ if (!FrameReg) {
+ FIOp->ChangeToImmediate(Offset);
+ if (TII->isImmOperandLegal(*MI, FIOperandNum, *FIOp))
+ return false;
+ }
+
+ // We need to use register here. Check if we can use an SGPR or need
+ // a VGPR.
+ FIOp->ChangeToRegister(AMDGPU::M0, false);
+ bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, FIOp);
+
+ if (!Offset && FrameReg && UseSGPR) {
+ FIOp->setReg(FrameReg);
+ return false;
+ }
+
+ const TargetRegisterClass *RC =
+ UseSGPR ? &AMDGPU::SReg_32_XM0RegClass : &AMDGPU::VGPR_32RegClass;
+
+ Register TmpReg =
+ RS->scavengeRegisterBackwards(*RC, MI, false, 0, !UseSGPR);
+ FIOp->setReg(TmpReg);
+ FIOp->setIsKill();
+
+ if ((!FrameReg || !Offset) && TmpReg) {
+ unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
+ auto MIB = BuildMI(*MBB, MI, DL, TII->get(Opc), TmpReg);
+ if (FrameReg)
+ MIB.addReg(FrameReg);
+ else
+ MIB.addImm(Offset);
+
+ return false;
+ }
+
+ bool NeedSaveSCC = RS->isRegUsed(AMDGPU::SCC) &&
+ !MI->definesRegister(AMDGPU::SCC, /*TRI=*/nullptr);
+
+ Register TmpSReg =
+ UseSGPR ? TmpReg
+ : RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
+ MI, false, 0, !UseSGPR);
+
+ if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR)) {
+ int SVOpcode = AMDGPU::getFlatScratchInstSVfromSS(MI->getOpcode());
+ if (ST.hasFlatScratchSVSMode() && SVOpcode != -1) {
+ Register TmpVGPR = RS->scavengeRegisterBackwards(
+ AMDGPU::VGPR_32RegClass, MI, false, 0, /*AllowSpill=*/true);
+
+ // Materialize the frame register.
+ auto MIB =
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR);
+ if (FrameReg)
+ MIB.addReg(FrameReg);
+ else
+ MIB.addImm(Offset);
+
+ // Add the offset to the frame register.
+ if (FrameReg && Offset)
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), FrameReg)
+ .addReg(FrameReg, RegState::Kill)
+ .addImm(Offset);
+
+ BuildMI(*MBB, MI, DL, TII->get(SVOpcode))
+ .add(MI->getOperand(0)) // $vdata
+ .addReg(TmpVGPR) // $vaddr
+ .addImm(0) // Offset
+ .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::cpol));
+ MI->eraseFromParent();
+ return true;
+ }
+ report_fatal_error("Cannot scavenge register in FI elimination!");
+ }
+
+ if (!TmpSReg) {
+ // Use frame register and restore it after.
+ TmpSReg = FrameReg;
+ FIOp->setReg(FrameReg);
+ FIOp->setIsKill(false);
+ }
+
+ if (NeedSaveSCC) {
+ assert(!(Offset & 0x1) && "Flat scratch offset must be aligned!");
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADDC_U32), TmpSReg)
+ .addReg(FrameReg)
+ .addImm(Offset);
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITCMP1_B32))
+ .addReg(TmpSReg)
+ .addImm(0);
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITSET0_B32), TmpSReg)
+ .addImm(0)
+ .addReg(TmpSReg);
+ } else {
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpSReg)
+ .addReg(FrameReg)
+ .addImm(Offset);
+ }
+
+ if (!UseSGPR)
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
+ .addReg(TmpSReg, RegState::Kill);
+
+ if (TmpSReg == FrameReg) {
+ // Undo frame register modification.
+ if (NeedSaveSCC &&
+ !MI->registerDefIsDead(AMDGPU::SCC, /*TRI=*/nullptr)) {
+ MachineBasicBlock::iterator I =
+ BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADDC_U32),
+ TmpSReg)
+ .addReg(FrameReg)
+ .addImm(-Offset);
+ I = BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITCMP1_B32))
+ .addReg(TmpSReg)
+ .addImm(0);
+ BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITSET0_B32),
+ TmpSReg)
+ .addImm(0)
+ .addReg(TmpSReg);
+ } else {
+ BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADD_I32),
+ FrameReg)
+ .addReg(FrameReg)
+ .addImm(-Offset);
+ }
+ }
+
+ return false;
+ }
+
+ bool IsMUBUF = TII->isMUBUF(*MI);
+
+ if (!IsMUBUF && !MFI->isBottomOfStack()) {
+ // Convert to a swizzled stack address by scaling by the wave size.
+ // In an entry function/kernel the offset is already swizzled.
+ bool IsSALU = isSGPRClass(TII->getRegClass(MI->getDesc(), FIOperandNum));
+ bool LiveSCC = RS->isRegUsed(AMDGPU::SCC) &&
+ !MI->definesRegister(AMDGPU::SCC, /*TRI=*/nullptr);
+ const TargetRegisterClass *RC = IsSALU && !LiveSCC
+ ? &AMDGPU::SReg_32RegClass
+ : &AMDGPU::VGPR_32RegClass;
+ bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32 ||
+ MI->getOpcode() == AMDGPU::V_MOV_B32_e64 ||
+ MI->getOpcode() == AMDGPU::S_MOV_B32;
+ Register ResultReg =
+ IsCopy ? MI->getOperand(0).getReg()
+ : RS->scavengeRegisterBackwards(*RC, MI, false, 0);
+
+ int64_t Offset = FrameInfo.getObjectOffset(Index);
+ if (Offset == 0) {
+ unsigned OpCode =
+ IsSALU && !LiveSCC ? AMDGPU::S_LSHR_B32 : AMDGPU::V_LSHRREV_B32_e64;
+ Register TmpResultReg = ResultReg;
+ if (IsSALU && LiveSCC) {
+ TmpResultReg = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass,
+ MI, false, 0);
+ }
+
+ auto Shift = BuildMI(*MBB, MI, DL, TII->get(OpCode), TmpResultReg);
+ if (OpCode == AMDGPU::V_LSHRREV_B32_e64)
+ // For V_LSHRREV, the operands are reversed (the shift count goes
+ // first).
+ Shift.addImm(ST.getWavefrontSizeLog2()).addReg(FrameReg);
+ else
+ Shift.addReg(FrameReg).addImm(ST.getWavefrontSizeLog2());
+ if (IsSALU && !LiveSCC)
+ Shift.getInstr()->getOperand(3).setIsDead(); // Mark SCC as dead.
+ if (IsSALU && LiveSCC) {
+ Register NewDest;
+ if (IsCopy) {
+ assert(ResultReg.isPhysical());
+ NewDest = ResultReg;
+ } else {
+ NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
+ Shift, false, 0);
+ }
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest)
+ .addReg(TmpResultReg);
+ ResultReg = NewDest;
+ }
+ } else {
+ MachineInstrBuilder MIB;
+ if (!IsSALU) {
+ if ((MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) !=
+ nullptr) {
+ // Reuse ResultReg in intermediate step.
+ Register ScaledReg = ResultReg;
+
+ BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64),
+ ScaledReg)
+ .addImm(ST.getWavefrontSizeLog2())
+ .addReg(FrameReg);
+
+ const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32;
+
+ // TODO: Fold if use instruction is another add of a constant.
+ if (IsVOP2 ||
+ AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) {
+ // FIXME: This can fail
+ MIB.addImm(Offset);
+ MIB.addReg(ScaledReg, RegState::Kill);
+ if (!IsVOP2)
+ MIB.addImm(0); // clamp bit
+ } else {
+ assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 &&
+ "Need to reuse carry out register");
+
+ // Use scavenged unused carry out as offset register.
+ Register ConstOffsetReg;
+ if (!isWave32)
+ ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0);
+ else
+ ConstOffsetReg = MIB.getReg(1);
+
+ BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32),
+ ConstOffsetReg)
+ .addImm(Offset);
+ MIB.addReg(ConstOffsetReg, RegState::Kill);
+ MIB.addReg(ScaledReg, RegState::Kill);
+ MIB.addImm(0); // clamp bit
+ }
+ }
+ }
+ if (!MIB || IsSALU) {
+ // We have to produce a carry out, and there isn't a free SGPR pair
+ // for it. We can keep the whole computation on the SALU to avoid
+ // clobbering an additional register at the cost of an extra mov.
+
+ // We may have 1 free scratch SGPR even though a carry out is
+ // unavailable. Only one additional mov is needed.
+ Register TmpScaledReg = IsCopy && IsSALU
+ ? ResultReg
+ : RS->scavengeRegisterBackwards(
+ AMDGPU::SReg_32_XM0RegClass, MI,
+ false, 0, /*AllowSpill=*/false);
+ Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg;
+ Register TmpResultReg = ScaledReg;
+
+ if (!LiveSCC) {
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), TmpResultReg)
+ .addReg(FrameReg)
+ .addImm(ST.getWavefrontSizeLog2());
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpResultReg)
+ .addReg(TmpResultReg, RegState::Kill)
+ .addImm(Offset);
+ } else {
+ TmpResultReg = RS->scavengeRegisterBackwards(
+ AMDGPU::VGPR_32RegClass, MI, false, 0, /*AllowSpill=*/true);
+
+ MachineInstrBuilder Add;
+ if ((Add = TII->getAddNoCarry(*MBB, MI, DL, TmpResultReg, *RS))) {
+ BuildMI(*MBB, *Add, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64),
+ TmpResultReg)
+ .addImm(ST.getWavefrontSizeLog2())
+ .addReg(FrameReg);
+ if (Add->getOpcode() == AMDGPU::V_ADD_CO_U32_e64) {
+ BuildMI(*MBB, *Add, DL, TII->get(AMDGPU::S_MOV_B32), ResultReg)
+ .addImm(Offset);
+ Add.addReg(ResultReg, RegState::Kill)
+ .addReg(TmpResultReg, RegState::Kill)
+ .addImm(0);
+ } else
+ Add.addImm(Offset).addReg(TmpResultReg, RegState::Kill);
+ } else {
+ assert(Offset > 0 && isUInt<24>(2 * ST.getMaxWaveScratchSize()) &&
+ "offset is unsafe for v_mad_u32_u24");
+
+ // We start with a frame pointer with a wave space value, and
+ // an offset in lane-space. We are materializing a lane space
+ // value. We can either do a right shift of the frame pointer
+ // to get to lane space, or a left shift of the offset to get
+ // to wavespace. We can right shift after the computation to
+ // get back to the desired per-lane value. We are using the
+ // mad_u32_u24 primarily as an add with no carry out clobber.
+ bool IsInlinableLiteral =
+ AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm());
+ if (!IsInlinableLiteral) {
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32),
+ TmpResultReg)
+ .addImm(Offset);
+ }
+
+ Add = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MAD_U32_U24_e64),
+ TmpResultReg);
+
+ if (!IsInlinableLiteral) {
+ Add.addReg(TmpResultReg, RegState::Kill);
+ } else {
+ // We fold the offset into mad itself if its inlinable.
+ Add.addImm(Offset);
+ }
+ Add.addImm(ST.getWavefrontSize()).addReg(FrameReg).addImm(0);
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64),
+ TmpResultReg)
+ .addImm(ST.getWavefrontSizeLog2())
+ .addReg(TmpResultReg);
+ }
+
+ Register NewDest;
+ if (IsCopy) {
+ NewDest = ResultReg;
+ } else {
+ NewDest = RS->scavengeRegisterBackwards(
+ AMDGPU::SReg_32_XM0RegClass, *Add, false, 0,
+ /*AllowSpill=*/true);
+ }
+
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
+ NewDest)
+ .addReg(TmpResultReg);
+ ResultReg = NewDest;
+ }
+ if (!IsSALU)
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg)
+ .addReg(TmpResultReg, RegState::Kill);
+ else
+ ResultReg = TmpResultReg;
+ // If there were truly no free SGPRs, we need to undo everything.
+ if (!TmpScaledReg.isValid()) {
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg)
+ .addReg(ScaledReg, RegState::Kill)
+ .addImm(-Offset);
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg)
+ .addReg(FrameReg)
+ .addImm(ST.getWavefrontSizeLog2());
+ }
+ }
+ }
+
+ // Don't introduce an extra copy if we're just materializing in a mov.
+ if (IsCopy) {
+ MI->eraseFromParent();
+ return true;
+ }
+ FIOp->ChangeToRegister(ResultReg, false, false, true);
+ return false;
+ }
+
+ if (IsMUBUF) {
+ // Disable offen so we don't need a 0 vgpr base.
+ assert(
+ static_cast<int>(FIOperandNum) ==
+ AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::vaddr));
+
+ auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset);
+ assert((SOffset.isImm() && SOffset.getImm() == 0));
+
+ if (FrameReg != AMDGPU::NoRegister)
+ SOffset.ChangeToRegister(FrameReg, false);
+
+ int64_t Offset = FrameInfo.getObjectOffset(Index);
+ int64_t OldImm =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm();
+ int64_t NewOffset = OldImm + Offset;
+
+ if (TII->isLegalMUBUFImmOffset(NewOffset) &&
+ buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) {
+ MI->eraseFromParent();
+ return true;
+ }
+ }
+
+ // If the offset is simply too big, don't convert to a scratch wave offset
+ // relative index.
+
+ FIOp->ChangeToImmediate(Offset);
+ if (!TII->isImmOperandLegal(*MI, FIOperandNum, *FIOp)) {
+ Register TmpReg =
+ RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0);
+ BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
+ .addImm(Offset);
+ FIOp->ChangeToRegister(TmpReg, false, false, true);
+ }
+
+ return false;
+}
+
+StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const {
+ return AMDGPUInstPrinter::getRegisterName(Reg);
+}
+
+unsigned SIRegisterInfo::getHWRegIndex(MCRegister Reg) const {
+ return getEncodingValue(Reg) & AMDGPU::HWEncoding::REG_IDX_MASK;
+}
+
+unsigned AMDGPU::getRegBitWidth(const TargetRegisterClass &RC) {
+ return getRegBitWidth(RC.getID());
+}
+
+static const TargetRegisterClass *
+getAnyVGPRClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::VReg_64RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::VReg_96RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::VReg_128RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::VReg_160RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::VReg_192RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::VReg_224RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::VReg_256RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::VReg_288RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::VReg_320RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::VReg_352RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::VReg_384RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::VReg_512RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::VReg_1024RegClass;
+
+ return nullptr;
+}
+
+static const TargetRegisterClass *
+getAlignedVGPRClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::VReg_64_Align2RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::VReg_96_Align2RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::VReg_128_Align2RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::VReg_160_Align2RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::VReg_192_Align2RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::VReg_224_Align2RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::VReg_256_Align2RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::VReg_288_Align2RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::VReg_320_Align2RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::VReg_352_Align2RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::VReg_384_Align2RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::VReg_512_Align2RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::VReg_1024_Align2RegClass;
+
+ return nullptr;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const {
+ if (BitWidth == 1)
+ return &AMDGPU::VReg_1RegClass;
+ if (BitWidth == 16)
+ return &AMDGPU::VGPR_16RegClass;
+ if (BitWidth == 32)
+ return &AMDGPU::VGPR_32RegClass;
+ return ST.needsAlignedVGPRs() ? getAlignedVGPRClassForBitWidth(BitWidth)
+ : getAnyVGPRClassForBitWidth(BitWidth);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getAlignedLo256VGPRClassForBitWidth(unsigned BitWidth) const {
+ if (BitWidth <= 32)
+ return &AMDGPU::VGPR_32_Lo256RegClass;
+ if (BitWidth <= 64)
+ return &AMDGPU::VReg_64_Lo256_Align2RegClass;
+ if (BitWidth <= 96)
+ return &AMDGPU::VReg_96_Lo256_Align2RegClass;
+ if (BitWidth <= 128)
+ return &AMDGPU::VReg_128_Lo256_Align2RegClass;
+ if (BitWidth <= 160)
+ return &AMDGPU::VReg_160_Lo256_Align2RegClass;
+ if (BitWidth <= 192)
+ return &AMDGPU::VReg_192_Lo256_Align2RegClass;
+ if (BitWidth <= 224)
+ return &AMDGPU::VReg_224_Lo256_Align2RegClass;
+ if (BitWidth <= 256)
+ return &AMDGPU::VReg_256_Lo256_Align2RegClass;
+ if (BitWidth <= 288)
+ return &AMDGPU::VReg_288_Lo256_Align2RegClass;
+ if (BitWidth <= 320)
+ return &AMDGPU::VReg_320_Lo256_Align2RegClass;
+ if (BitWidth <= 352)
+ return &AMDGPU::VReg_352_Lo256_Align2RegClass;
+ if (BitWidth <= 384)
+ return &AMDGPU::VReg_384_Lo256_Align2RegClass;
+ if (BitWidth <= 512)
+ return &AMDGPU::VReg_512_Lo256_Align2RegClass;
+ if (BitWidth <= 1024)
+ return &AMDGPU::VReg_1024_Lo256_Align2RegClass;
+
+ return nullptr;
+}
+
+static const TargetRegisterClass *
+getAnyAGPRClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::AReg_64RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::AReg_96RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::AReg_128RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::AReg_160RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::AReg_192RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::AReg_224RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::AReg_256RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::AReg_288RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::AReg_320RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::AReg_352RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::AReg_384RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::AReg_512RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::AReg_1024RegClass;
+
+ return nullptr;
+}
+
+static const TargetRegisterClass *
+getAlignedAGPRClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::AReg_64_Align2RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::AReg_96_Align2RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::AReg_128_Align2RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::AReg_160_Align2RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::AReg_192_Align2RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::AReg_224_Align2RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::AReg_256_Align2RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::AReg_288_Align2RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::AReg_320_Align2RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::AReg_352_Align2RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::AReg_384_Align2RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::AReg_512_Align2RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::AReg_1024_Align2RegClass;
+
+ return nullptr;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const {
+ if (BitWidth == 16)
+ return &AMDGPU::AGPR_LO16RegClass;
+ if (BitWidth == 32)
+ return &AMDGPU::AGPR_32RegClass;
+ return ST.needsAlignedVGPRs() ? getAlignedAGPRClassForBitWidth(BitWidth)
+ : getAnyAGPRClassForBitWidth(BitWidth);
+}
+
+static const TargetRegisterClass *
+getAnyVectorSuperClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::AV_64RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::AV_96RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::AV_128RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::AV_160RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::AV_192RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::AV_224RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::AV_256RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::AV_288RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::AV_320RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::AV_352RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::AV_384RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::AV_512RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::AV_1024RegClass;
+
+ return nullptr;
+}
+
+static const TargetRegisterClass *
+getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 64)
+ return &AMDGPU::AV_64_Align2RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::AV_96_Align2RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::AV_128_Align2RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::AV_160_Align2RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::AV_192_Align2RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::AV_224_Align2RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::AV_256_Align2RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::AV_288_Align2RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::AV_320_Align2RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::AV_352_Align2RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::AV_384_Align2RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::AV_512_Align2RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::AV_1024_Align2RegClass;
+
+ return nullptr;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const {
+ if (BitWidth == 32)
+ return &AMDGPU::AV_32RegClass;
+ return ST.needsAlignedVGPRs()
+ ? getAlignedVectorSuperClassForBitWidth(BitWidth)
+ : getAnyVectorSuperClassForBitWidth(BitWidth);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getDefaultVectorSuperClassForBitWidth(unsigned BitWidth) const {
+ // TODO: In principle this should use AV classes for gfx908 too. This is
+ // limited to 90a+ to avoid regressing special case copy optimizations which
+ // need new handling. The core issue is that it's not possible to directly
+ // copy between AGPRs on gfx908, and the current optimizations around that
+ // expect to see copies to VGPR.
+ return ST.hasGFX90AInsts() ? getVectorSuperClassForBitWidth(BitWidth)
+ : getVGPRClassForBitWidth(BitWidth);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) {
+ if (BitWidth == 16 || BitWidth == 32)
+ return &AMDGPU::SReg_32RegClass;
+ if (BitWidth == 64)
+ return &AMDGPU::SReg_64RegClass;
+ if (BitWidth == 96)
+ return &AMDGPU::SGPR_96RegClass;
+ if (BitWidth == 128)
+ return &AMDGPU::SGPR_128RegClass;
+ if (BitWidth == 160)
+ return &AMDGPU::SGPR_160RegClass;
+ if (BitWidth == 192)
+ return &AMDGPU::SGPR_192RegClass;
+ if (BitWidth == 224)
+ return &AMDGPU::SGPR_224RegClass;
+ if (BitWidth == 256)
+ return &AMDGPU::SGPR_256RegClass;
+ if (BitWidth == 288)
+ return &AMDGPU::SGPR_288RegClass;
+ if (BitWidth == 320)
+ return &AMDGPU::SGPR_320RegClass;
+ if (BitWidth == 352)
+ return &AMDGPU::SGPR_352RegClass;
+ if (BitWidth == 384)
+ return &AMDGPU::SGPR_384RegClass;
+ if (BitWidth == 512)
+ return &AMDGPU::SGPR_512RegClass;
+ if (BitWidth == 1024)
+ return &AMDGPU::SGPR_1024RegClass;
+
+ return nullptr;
+}
+
+bool SIRegisterInfo::isSGPRReg(const MachineRegisterInfo &MRI,
+ Register Reg) const {
+ const TargetRegisterClass *RC;
+ if (Reg.isVirtual())
+ RC = MRI.getRegClass(Reg);
+ else
+ RC = getPhysRegBaseClass(Reg);
+ return RC && isSGPRClass(RC);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const {
+ unsigned Size = getRegSizeInBits(*SRC);
+
+ switch (SRC->getID()) {
+ default:
+ break;
+ case AMDGPU::VS_32_Lo256RegClassID:
+ case AMDGPU::VS_64_Lo256RegClassID:
+ return getAllocatableClass(getAlignedLo256VGPRClassForBitWidth(Size));
+ }
+
+ const TargetRegisterClass *VRC =
+ getAllocatableClass(getVGPRClassForBitWidth(Size));
+ assert(VRC && "Invalid register class size");
+ return VRC;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const {
+ unsigned Size = getRegSizeInBits(*SRC);
+ const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size);
+ assert(ARC && "Invalid register class size");
+ return ARC;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getEquivalentAVClass(const TargetRegisterClass *SRC) const {
+ unsigned Size = getRegSizeInBits(*SRC);
+ const TargetRegisterClass *ARC = getVectorSuperClassForBitWidth(Size);
+ assert(ARC && "Invalid register class size");
+ return ARC;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const {
+ unsigned Size = getRegSizeInBits(*VRC);
+ if (Size == 32)
+ return &AMDGPU::SGPR_32RegClass;
+ const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size);
+ assert(SRC && "Invalid register class size");
+ return SRC;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getCompatibleSubRegClass(const TargetRegisterClass *SuperRC,
+ const TargetRegisterClass *SubRC,
+ unsigned SubIdx) const {
+ // Ensure this subregister index is aligned in the super register.
+ const TargetRegisterClass *MatchRC =
+ getMatchingSuperRegClass(SuperRC, SubRC, SubIdx);
+ return MatchRC && MatchRC->hasSubClassEq(SuperRC) ? MatchRC : nullptr;
+}
+
+bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const {
+ if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&
+ OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST)
+ return !ST.hasMFMAInlineLiteralBug();
+
+ return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
+ OpType <= AMDGPU::OPERAND_SRC_LAST;
+}
+
+bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const {
+ // TODO: 64-bit operands have extending behavior from 32-bit literal.
+ return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST &&
+ OpType <= AMDGPU::OPERAND_REG_IMM_LAST;
+}
+
+/// Returns a lowest register that is not used at any point in the function.
+/// If all registers are used, then this function will return
+/// AMDGPU::NoRegister. If \p ReserveHighestRegister = true, then return
+/// highest unused register.
+MCRegister SIRegisterInfo::findUnusedRegister(
+ const MachineRegisterInfo &MRI, const TargetRegisterClass *RC,
+ const MachineFunction &MF, bool ReserveHighestRegister) const {
+ if (ReserveHighestRegister) {
+ for (MCRegister Reg : reverse(*RC))
+ if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
+ return Reg;
+ } else {
+ for (MCRegister Reg : *RC)
+ if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
+ return Reg;
+ }
+ return MCRegister();
+}
+
+bool SIRegisterInfo::isUniformReg(const MachineRegisterInfo &MRI,
+ const RegisterBankInfo &RBI,
+ Register Reg) const {
+ auto *RB = RBI.getRegBank(Reg, MRI, *MRI.getTargetRegisterInfo());
+ if (!RB)
+ return false;
+
+ return !RBI.isDivergentRegBank(RB);
+}
+
+ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC,
+ unsigned EltSize) const {
+ const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC);
+ assert(RegBitWidth >= 32 && RegBitWidth <= 1024 && EltSize >= 2);
+
+ const unsigned RegHalves = RegBitWidth / 16;
+ const unsigned EltHalves = EltSize / 2;
+ assert(RegSplitParts.size() + 1 >= EltHalves);
+
+ const std::vector<int16_t> &Parts = RegSplitParts[EltHalves - 1];
+ const unsigned NumParts = RegHalves / EltHalves;
+
+ return ArrayRef(Parts.data(), NumParts);
+}
+
+const TargetRegisterClass*
+SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI,
+ Register Reg) const {
+ return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegBaseClass(Reg);
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getRegClassForOperandReg(const MachineRegisterInfo &MRI,
+ const MachineOperand &MO) const {
+ const TargetRegisterClass *SrcRC = getRegClassForReg(MRI, MO.getReg());
+ return getSubRegisterClass(SrcRC, MO.getSubReg());
+}
+
+bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI,
+ Register Reg) const {
+ const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
+ // Registers without classes are unaddressable, SGPR-like registers.
+ return RC && isVGPRClass(RC);
+}
+
+bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI,
+ Register Reg) const {
+ const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
+
+ // Registers without classes are unaddressable, SGPR-like registers.
+ return RC && isAGPRClass(RC);
+}
+
+unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
+ MachineFunction &MF) const {
+ unsigned MinOcc = ST.getOccupancyWithWorkGroupSizes(MF).first;
+ switch (RC->getID()) {
+ default:
+ return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF);
+ case AMDGPU::VGPR_32RegClassID:
+ return std::min(
+ ST.getMaxNumVGPRs(
+ MinOcc,
+ MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize()),
+ ST.getMaxNumVGPRs(MF));
+ case AMDGPU::SGPR_32RegClassID:
+ case AMDGPU::SGPR_LO16RegClassID:
+ return std::min(ST.getMaxNumSGPRs(MinOcc, true), ST.getMaxNumSGPRs(MF));
+ }
+}
+
+unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF,
+ unsigned Idx) const {
+ switch (static_cast<AMDGPU::RegisterPressureSets>(Idx)) {
+ case AMDGPU::RegisterPressureSets::VGPR_32:
+ case AMDGPU::RegisterPressureSets::AGPR_32:
+ return getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
+ const_cast<MachineFunction &>(MF));
+ case AMDGPU::RegisterPressureSets::SReg_32:
+ return getRegPressureLimit(&AMDGPU::SGPR_32RegClass,
+ const_cast<MachineFunction &>(MF));
+ }
+
+ llvm_unreachable("Unexpected register pressure set!");
+}
+
+const int *SIRegisterInfo::getRegUnitPressureSets(MCRegUnit RegUnit) const {
+ static const int Empty[] = { -1 };
+
+ if (RegPressureIgnoredUnits[static_cast<unsigned>(RegUnit)])
+ return Empty;
+
+ return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit);
+}
+
+bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
+ ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &Hints,
+ const MachineFunction &MF,
+ const VirtRegMap *VRM,
+ const LiveRegMatrix *Matrix) const {
+
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+
+ std::pair<unsigned, Register> Hint = MRI.getRegAllocationHint(VirtReg);
+
+ switch (Hint.first) {
+ case AMDGPURI::Size32: {
+ Register Paired = Hint.second;
+ assert(Paired);
+ Register PairedPhys;
+ if (Paired.isPhysical()) {
+ PairedPhys =
+ getMatchingSuperReg(Paired, AMDGPU::lo16, &AMDGPU::VGPR_32RegClass);
+ } else if (VRM && VRM->hasPhys(Paired)) {
+ PairedPhys = getMatchingSuperReg(VRM->getPhys(Paired), AMDGPU::lo16,
+ &AMDGPU::VGPR_32RegClass);
+ }
+
+ // Prefer the paired physreg.
+ if (PairedPhys)
+ // isLo(Paired) is implicitly true here from the API of
+ // getMatchingSuperReg.
+ Hints.push_back(PairedPhys);
+ return false;
+ }
+ case AMDGPURI::Size16: {
+ Register Paired = Hint.second;
+ assert(Paired);
+ Register PairedPhys;
+ if (Paired.isPhysical()) {
+ PairedPhys = TRI->getSubReg(Paired, AMDGPU::lo16);
+ } else if (VRM && VRM->hasPhys(Paired)) {
+ PairedPhys = TRI->getSubReg(VRM->getPhys(Paired), AMDGPU::lo16);
+ }
+
+ // First prefer the paired physreg.
+ if (PairedPhys)
+ Hints.push_back(PairedPhys);
+ else {
+ // Add all the lo16 physregs.
+ // When the Paired operand has not yet been assigned a physreg it is
+ // better to try putting VirtReg in a lo16 register, because possibly
+ // later Paired can be assigned to the overlapping register and the COPY
+ // can be eliminated.
+ for (MCPhysReg PhysReg : Order) {
+ if (PhysReg == PairedPhys || AMDGPU::isHi16Reg(PhysReg, *this))
+ continue;
+ if (AMDGPU::VGPR_16RegClass.contains(PhysReg) &&
+ !MRI.isReserved(PhysReg))
+ Hints.push_back(PhysReg);
+ }
+ }
+ return false;
+ }
+ default:
+ return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
+ VRM);
+ }
+}
+
+MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
+ // Not a callee saved register.
+ return AMDGPU::SGPR30_SGPR31;
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
+ const RegisterBank &RB) const {
+ switch (RB.getID()) {
+ case AMDGPU::VGPRRegBankID:
+ return getVGPRClassForBitWidth(
+ std::max(ST.useRealTrue16Insts() ? 16u : 32u, Size));
+ case AMDGPU::VCCRegBankID:
+ assert(Size == 1);
+ return getWaveMaskRegClass();
+ case AMDGPU::SGPRRegBankID:
+ return getSGPRClassForBitWidth(std::max(32u, Size));
+ case AMDGPU::AGPRRegBankID:
+ return getAGPRClassForBitWidth(std::max(32u, Size));
+ default:
+ llvm_unreachable("unknown register bank");
+ }
+}
+
+const TargetRegisterClass *
+SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
+ const MachineRegisterInfo &MRI) const {
+ const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg());
+ if (const RegisterBank *RB = dyn_cast<const RegisterBank *>(RCOrRB))
+ return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB);
+
+ if (const auto *RC = dyn_cast<const TargetRegisterClass *>(RCOrRB))
+ return getAllocatableClass(RC);
+
+ return nullptr;
+}
+
+MCRegister SIRegisterInfo::getVCC() const {
+ return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC;
+}
+
+MCRegister SIRegisterInfo::getExec() const {
+ return isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+}
+
+const TargetRegisterClass *SIRegisterInfo::getVGPR64Class() const {
+ // VGPR tuples have an alignment requirement on gfx90a variants.
+ return ST.needsAlignedVGPRs() ? &AMDGPU::VReg_64_Align2RegClass
+ : &AMDGPU::VReg_64RegClass;
+}
+
+// Find reaching register definition
+MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg,
+ MachineInstr &Use,
+ MachineRegisterInfo &MRI,
+ LiveIntervals *LIS) const {
+ auto &MDT = LIS->getDomTree();
+ SlotIndex UseIdx = LIS->getInstructionIndex(Use);
+ SlotIndex DefIdx;
+
+ if (Reg.isVirtual()) {
+ if (!LIS->hasInterval(Reg))
+ return nullptr;
+ LiveInterval &LI = LIS->getInterval(Reg);
+ LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg)
+ : MRI.getMaxLaneMaskForVReg(Reg);
+ VNInfo *V = nullptr;
+ if (LI.hasSubRanges()) {
+ for (auto &S : LI.subranges()) {
+ if ((S.LaneMask & SubLanes) == SubLanes) {
+ V = S.getVNInfoAt(UseIdx);
+ break;
+ }
+ }
+ } else {
+ V = LI.getVNInfoAt(UseIdx);
+ }
+ if (!V)
+ return nullptr;
+ DefIdx = V->def;
+ } else {
+ // Find last def.
+ for (MCRegUnit Unit : regunits(Reg.asMCReg())) {
+ LiveRange &LR = LIS->getRegUnit(Unit);
+ if (VNInfo *V = LR.getVNInfoAt(UseIdx)) {
+ if (!DefIdx.isValid() ||
+ MDT.dominates(LIS->getInstructionFromIndex(DefIdx),
+ LIS->getInstructionFromIndex(V->def)))
+ DefIdx = V->def;
+ } else {
+ return nullptr;
+ }
+ }
+ }
+
+ MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx);
+
+ if (!Def || !MDT.dominates(Def, &Use))
+ return nullptr;
+
+ assert(Def->modifiesRegister(Reg, this));
+
+ return Def;
+}
+
+MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const {
+ assert(getRegSizeInBits(*getPhysRegBaseClass(Reg)) <= 32);
+
+ for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass,
+ AMDGPU::SReg_32RegClass,
+ AMDGPU::AGPR_32RegClass } ) {
+ if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC))
+ return Super;
+ }
+ if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16,
+ &AMDGPU::VGPR_32RegClass)) {
+ return Super;
+ }
+
+ return AMDGPU::NoRegister;
+}
+
+bool SIRegisterInfo::isProperlyAlignedRC(const TargetRegisterClass &RC) const {
+ if (!ST.needsAlignedVGPRs())
+ return true;
+
+ if (isVGPRClass(&RC))
+ return RC.hasSuperClassEq(getVGPRClassForBitWidth(getRegSizeInBits(RC)));
+ if (isAGPRClass(&RC))
+ return RC.hasSuperClassEq(getAGPRClassForBitWidth(getRegSizeInBits(RC)));
+ if (isVectorSuperClass(&RC))
+ return RC.hasSuperClassEq(
+ getVectorSuperClassForBitWidth(getRegSizeInBits(RC)));
+
+ assert(&RC != &AMDGPU::VS_64RegClass);
+
+ return true;
+}
+
+ArrayRef<MCPhysReg>
+SIRegisterInfo::getAllSGPR128(const MachineFunction &MF) const {
+ return ArrayRef(AMDGPU::SGPR_128RegClass.begin(), ST.getMaxNumSGPRs(MF) / 4);
+}
+
+ArrayRef<MCPhysReg>
+SIRegisterInfo::getAllSGPR64(const MachineFunction &MF) const {
+ return ArrayRef(AMDGPU::SGPR_64RegClass.begin(), ST.getMaxNumSGPRs(MF) / 2);
+}
+
+ArrayRef<MCPhysReg>
+SIRegisterInfo::getAllSGPR32(const MachineFunction &MF) const {
+ return ArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF));
+}
+
+unsigned
+SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
+ unsigned SubReg) const {
+ switch (RC->TSFlags & SIRCFlags::RegKindMask) {
+ case SIRCFlags::HasSGPR:
+ return std::min(128u, getSubRegIdxSize(SubReg));
+ case SIRCFlags::HasAGPR:
+ case SIRCFlags::HasVGPR:
+ case SIRCFlags::HasVGPR | SIRCFlags::HasAGPR:
+ return std::min(32u, getSubRegIdxSize(SubReg));
+ default:
+ break;
+ }
+ return 0;
+}
+
+unsigned SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
+ const TargetRegisterClass &RC,
+ bool IncludeCalls) const {
+ unsigned NumArchVGPRs = ST.has1024AddressableVGPRs() ? 1024 : 256;
+ ArrayRef<MCPhysReg> Registers =
+ (RC.getID() == AMDGPU::VGPR_32RegClassID)
+ ? RC.getRegisters().take_front(NumArchVGPRs)
+ : RC.getRegisters();
+ for (MCPhysReg Reg : reverse(Registers))
+ if (MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/!IncludeCalls))
+ return getHWRegIndex(Reg) + 1;
+ return 0;
+}
+
+SmallVector<StringLiteral>
+SIRegisterInfo::getVRegFlagsOfReg(Register Reg,
+ const MachineFunction &MF) const {
+ SmallVector<StringLiteral> RegFlags;
+ const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+ if (FuncInfo->checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))
+ RegFlags.push_back("WWM_REG");
+ return RegFlags;
+}
diff --git a/llvm/lib/Target/AMDGPU/.#SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/.#SIRegisterInfo.cpp
new file mode 120000
index 0000000000000..f0eff68ff4275
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/.#SIRegisterInfo.cpp
@@ -0,0 +1 @@
+cdevadas at hy-7c0110-rsqa-03.60790:1763344941
\ No newline at end of file
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 83bbff593cd38..0b9f56c1bcd44 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1878,7 +1878,7 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DestReg, int FrameIndex,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction *MF = MBB.getParent();
SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
@@ -1886,12 +1886,23 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
const DebugLoc &DL = MBB.findDebugLoc(MI);
unsigned SpillSize = RI.getSpillSize(*RC);
+ unsigned SubRegIdx = 0;
+ if (SubReg) {
+ uint64_t Mask = RI.getSubRegIndexLaneMask(SubReg).getAsInteger();
+ assert(llvm::popcount(Mask) % 2 == 0 &&
+ "expected only 32-bit subreg access");
+
+ // For subreg reload, identify the start offset. Each 32-bit register
+ // consists of two regunits and eventually two bits in the Lanemask.
+ SubRegIdx = llvm::countr_zero(Mask) / 2;
+ }
+
MachinePointerInfo PtrInfo
= MachinePointerInfo::getFixedStack(*MF, FrameIndex);
- MachineMemOperand *MMO = MF->getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
- FrameInfo.getObjectAlign(FrameIndex));
+ MachineMemOperand *MMO =
+ MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, SpillSize,
+ FrameInfo.getObjectAlign(FrameIndex));
if (RI.isSGPRClass(RC)) {
MFI->setHasSpilledSGPRs();
@@ -1911,19 +1922,22 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
BuildMI(MBB, MI, DL, OpDesc, DestReg)
.addFrameIndex(FrameIndex) // addr
- .addImm(0) // offset
+ .addImm(SubRegIdx) // offset
.addMemOperand(MMO)
.addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
return;
}
+ // Convert the subreg index to stack offset.
+ SubRegIdx *= 4;
+
unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC,
SpillSize, *MFI);
BuildMI(MBB, MI, DL, get(Opcode), DestReg)
.addFrameIndex(FrameIndex) // vaddr
.addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
- .addImm(0) // offset
+ .addImm(SubRegIdx) // offset
.addMemOperand(MMO);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 9d44950368630..1e46fe0e2d511 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -315,6 +315,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool expandPostRAPseudo(MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 4777e06a001f0..d7d56482ea4d3 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3634,6 +3634,15 @@ bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI,
return RC && isAGPRClass(RC);
}
+bool SIRegisterInfo::shouldEnableSubRegReload(unsigned SubReg) const {
+ // Disable lo16 and hi16 (16-bit) accesses as they are subreg views of the
+ // same 32-bit register and don't represent independent storage. If the number
+ // of bits set in the mask is odd, it indicates the presence of a 16-bit
+ // access as each 32-bit register consists of two Regunits and they take two
+ // bits in the regmask.
+ return getSubRegIndexLaneMask(SubReg).getNumLanes() % 2 == 0;
+}
+
unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
MachineFunction &MF) const {
unsigned MinOcc = ST.getOccupancyWithWorkGroupSizes(MF).first;
@@ -3767,6 +3776,11 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
}
}
+const TargetRegisterClass *
+SIRegisterInfo::getConstrainedRegClass(const TargetRegisterClass *RC) const {
+ return getProperlyAlignedRC(RC);
+}
+
const TargetRegisterClass *
SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 4c8e217aa5007..60cc776180816 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -346,6 +346,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
ArrayRef<int16_t> getRegSplitParts(const TargetRegisterClass *RC,
unsigned EltSize) const;
+ bool shouldEnableSubRegReload(unsigned SubReg) const override;
+
unsigned getRegPressureLimit(const TargetRegisterClass *RC,
MachineFunction &MF) const override;
@@ -369,6 +371,9 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
return getRegClassForSizeOnBank(Ty.getSizeInBits(), Bank);
}
+ const TargetRegisterClass *
+ getConstrainedRegClass(const TargetRegisterClass *RC) const override;
+
const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const override;
diff --git a/llvm/lib/Target/ARC/ARCInstrInfo.cpp b/llvm/lib/Target/ARC/ARCInstrInfo.cpp
index e17ecbf87faae..f43dc550ec5d5 100644
--- a/llvm/lib/Target/ARC/ARCInstrInfo.cpp
+++ b/llvm/lib/Target/ARC/ARCInstrInfo.cpp
@@ -323,7 +323,7 @@ void ARCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FrameIndex,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL = MBB.findDebugLoc(I);
MachineFunction &MF = *MBB.getParent();
diff --git a/llvm/lib/Target/ARC/ARCInstrInfo.h b/llvm/lib/Target/ARC/ARCInstrInfo.h
index ebeaf877f8436..03f6977c8f94a 100644
--- a/llvm/lib/Target/ARC/ARCInstrInfo.h
+++ b/llvm/lib/Target/ARC/ARCInstrInfo.h
@@ -76,6 +76,7 @@ class ARCInstrInfo : public ARCGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned subReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 02887ce93c525..402a4e30fe3ca 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -1212,7 +1212,7 @@ void ARMBaseInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end()) DL = I->getDebugLoc();
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 04e2ab055cf1a..ab94a113233dc 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -222,7 +222,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool expandPostRAPseudo(MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp b/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
index 01f588f0cdc38..c2345f588b937 100644
--- a/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -145,7 +145,7 @@ void Thumb1InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
assert((RC->hasSuperClassEq(&ARM::tGPRRegClass) ||
(DestReg.isPhysical() && isARMLowRegister(DestReg))) &&
diff --git a/llvm/lib/Target/ARM/Thumb1InstrInfo.h b/llvm/lib/Target/ARM/Thumb1InstrInfo.h
index 289a30a4ca1e4..53add79a508ff 100644
--- a/llvm/lib/Target/ARM/Thumb1InstrInfo.h
+++ b/llvm/lib/Target/ARM/Thumb1InstrInfo.h
@@ -49,7 +49,7 @@ class Thumb1InstrInfo : public ARMBaseInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool canCopyGluedNodeDuringSchedule(SDNode *N) const override;
diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
index efb92c9bcac18..76da9692fd9f6 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -210,7 +210,7 @@ void Thumb2InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.h b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
index 1e11cb37efc05..c6d38766f6d73 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.h
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
@@ -50,7 +50,7 @@ class Thumb2InstrInfo : public ARMBaseInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
/// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
index 6c37ba1411dde..3d9fa1854719f 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.cpp
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
@@ -160,7 +160,7 @@ void AVRInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DestReg, int FrameIndex,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction &MF = *MBB.getParent();
const MachineFrameInfo &MFI = MF.getFrameInfo();
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.h b/llvm/lib/Target/AVR/AVRInstrInfo.h
index 4db535a990451..8404b1686d88d 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.h
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.h
@@ -84,6 +84,7 @@ class AVRInstrInfo : public AVRGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
Register isLoadFromStackSlot(const MachineInstr &MI,
int &FrameIndex) const override;
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
index 095e2497eec17..b9eb601305fc9 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
@@ -151,7 +151,7 @@ void BPFInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end())
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.h b/llvm/lib/Target/BPF/BPFInstrInfo.h
index d3ef9bc164f4a..d9a9af1ebd743 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.h
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.h
@@ -45,7 +45,7 @@ class BPFInstrInfo : public BPFGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
MachineBasicBlock *&FBB,
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
index 3ab09902be3aa..904cdc7e2766e 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
@@ -438,7 +438,7 @@ void CSKYInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end())
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.h b/llvm/lib/Target/CSKY/CSKYInstrInfo.h
index d1cd0395f3b95..b5e3ee947dd68 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo.h
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.h
@@ -49,6 +49,7 @@ class CSKYInstrInfo : public CSKYGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 7682af4543b7c..f8929e1611564 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -1012,7 +1012,7 @@ void HexagonInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL = MBB.findDebugLoc(I);
MachineFunction &MF = *MBB.getParent();
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 796b978a2c3f0..fc96fb4f0834b 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -197,7 +197,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
/// This function is called for all pseudo instructions
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index 14b7557e7f94a..67f429c0ea4bc 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -70,7 +70,7 @@ void LanaiInstrInfo::loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator Position,
Register DestinationRegister, int FrameIndex,
const TargetRegisterClass *RegisterClass, Register /*VReg*/,
- MachineInstr::MIFlag /*Flags*/) const {
+ unsigned /*SubReg*/, MachineInstr::MIFlag /*Flags*/) const {
DebugLoc DL;
if (Position != MBB.end()) {
DL = Position->getDebugLoc();
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index 155e2f03be630..01b7d8a802805 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -65,6 +65,7 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
MachineBasicBlock &MBB, MachineBasicBlock::iterator Position,
Register DestinationRegister, int FrameIndex,
const TargetRegisterClass *RegisterClass, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool expandPostRAPseudo(MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
index 9fc862af7ea24..69ef349042d8a 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
@@ -149,7 +149,7 @@ void LoongArchInstrInfo::storeRegToStackSlot(
void LoongArchInstrInfo::loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register DstReg,
- int FI, const TargetRegisterClass *RC, Register VReg,
+ int FI, const TargetRegisterClass *RC, Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction *MF = MBB.getParent();
MachineFrameInfo &MFI = MF->getFrameInfo();
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h
index 9f7a0a2239a87..c25906e082c1c 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h
@@ -45,6 +45,7 @@ class LoongArchInstrInfo : public LoongArchGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
// Materializes the given integer Val into DstReg.
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 3805a18c4f35c..3a9f88269f42c 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -841,7 +841,7 @@ void M68kInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DstReg, int FrameIndex,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
const MachineFrameInfo &MFI = MBB.getParent()->getFrameInfo();
assert(MFI.getObjectSize(FrameIndex) >= TRI.getSpillSize(*RC) &&
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.h b/llvm/lib/Target/M68k/M68kInstrInfo.h
index e7ee117ee9941..c88c2f53f5f89 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.h
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.h
@@ -286,6 +286,7 @@ class M68kInstrInfo : public M68kGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool expandPostRAPseudo(MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp b/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
index 0fb4e9d9fcb62..14a863b2407db 100644
--- a/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -60,7 +60,7 @@ void MSP430InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DestReg, int FrameIdx,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (MI != MBB.end()) DL = MI->getDebugLoc();
diff --git a/llvm/lib/Target/MSP430/MSP430InstrInfo.h b/llvm/lib/Target/MSP430/MSP430InstrInfo.h
index c0a398452ef6d..546b2f45d9ba7 100644
--- a/llvm/lib/Target/MSP430/MSP430InstrInfo.h
+++ b/llvm/lib/Target/MSP430/MSP430InstrInfo.h
@@ -47,6 +47,7 @@ class MSP430InstrInfo : public MSP430GenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
int FrameIdx, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.h b/llvm/lib/Target/Mips/MipsInstrInfo.h
index 0b90972977d5e..29dfa0c41ad1d 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.h
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.h
@@ -155,7 +155,7 @@ class MipsInstrInfo : public MipsGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override {
loadRegFromStack(MBB, MBBI, DestReg, FrameIndex, RC, 0, Flags);
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
index 23889531431ea..3c150b1e26785 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
@@ -47,7 +47,7 @@ class NVPTXInstrInfo : public NVPTXGenInstrInfo {
* virtual void loadRegFromStackSlot(
* MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
* unsigned DestReg, int FrameIndex, const TargetRegisterClass *RC,
- * const TargetRegisterInfo *TRI, Register VReg,
+ * const TargetRegisterInfo *TRI, Register VReg, unsigned SubReg = 0,
* MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const;
*/
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 58ad12e2ce65a..9467c15ec6f64 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -2081,7 +2081,7 @@ void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DestReg, int FrameIdx,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
// We need to avoid a situation in which the value from a VRRC register is
// spilled using an Altivec instruction and reloaded into a VSRC register
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 8b824bc219ab2..af7064116ec8b 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -585,8 +585,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
-
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
// Emits a register reload without updating the register class for vector
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6a66c90d12463..647e2936cb1df 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -359,10 +359,10 @@ static void emitSiFiveCLICPreemptibleRestores(MachineFunction &MF,
// saved onto the stack in `emitSiFiveCLICPreemptibleSaves`.
TII->loadRegFromStackSlot(
MBB, MBBI, RISCV::X9, RVFI->getInterruptCSRFrameIndex(1),
- &RISCV::GPRRegClass, Register(), MachineInstr::FrameSetup);
+ &RISCV::GPRRegClass, Register(), /*SubReg=*/0, MachineInstr::FrameSetup);
TII->loadRegFromStackSlot(
MBB, MBBI, RISCV::X8, RVFI->getInterruptCSRFrameIndex(0),
- &RISCV::GPRRegClass, Register(), MachineInstr::FrameSetup);
+ &RISCV::GPRRegClass, Register(), /*SubReg=*/0, MachineInstr::FrameSetup);
}
// Get the ID of the libcall used for spilling and restoring callee saved
@@ -2257,7 +2257,7 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
MCRegister Reg = CS.getReg();
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, Register(),
- MachineInstr::FrameDestroy);
+ /*SubReg=*/0, MachineInstr::FrameDestroy);
assert(MI != MBB.begin() &&
"loadRegFromStackSlot didn't insert any code!");
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 5a717654e385c..847a9cdbe9a5b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -732,7 +732,7 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DstReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
MachineFunction *MF = MBB.getParent();
MachineFrameInfo &MFI = MF->getFrameInfo();
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 741a514d82bdd..05f97332a600f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -123,6 +123,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg,
int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
using TargetInstrInfo::foldMemoryOperandImpl;
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index 07f772a533df0..34f3118f314e5 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -567,7 +567,7 @@ void SparcInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end()) DL = I->getDebugLoc();
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.h b/llvm/lib/Target/Sparc/SparcInstrInfo.h
index 8f057018012b7..033af93777ca3 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.h
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.h
@@ -98,7 +98,7 @@ class SparcInstrInfo : public SparcGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
Register getGlobalBaseReg(MachineFunction *MF) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index db4f9a15d6497..43d0446fda2f2 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -1040,7 +1040,7 @@ void SystemZInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIdx,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
index 9fadf7bfb6d2b..6fc66c72a25e6 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -282,8 +282,7 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIdx, const TargetRegisterClass *RC,
-
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
LiveIntervals *LIS) const override;
diff --git a/llvm/lib/Target/VE/VEInstrInfo.cpp b/llvm/lib/Target/VE/VEInstrInfo.cpp
index b9ac5d6254362..3538dd0b521ab 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.cpp
+++ b/llvm/lib/Target/VE/VEInstrInfo.cpp
@@ -522,7 +522,7 @@ void VEInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FI,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end())
diff --git a/llvm/lib/Target/VE/VEInstrInfo.h b/llvm/lib/Target/VE/VEInstrInfo.h
index cedf7f21011ff..64503cda301c3 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.h
+++ b/llvm/lib/Target/VE/VEInstrInfo.h
@@ -99,8 +99,7 @@ class VEInstrInfo : public VEGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
-
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
/// } Stack Spill & Reload
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index ebed733314c6a..53b148c11c4e1 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4811,7 +4811,7 @@ void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register DestReg, int FrameIdx,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
const MachineFunction &MF = *MBB.getParent();
const MachineFrameInfo &MFI = MF.getFrameInfo();
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index a547fcd421411..de8ccb44578a3 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -472,9 +472,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
- int FrameIndex, const TargetRegisterClass *RC,
-
- Register VReg,
+ int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void loadStoreTileReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
diff --git a/llvm/lib/Target/XCore/XCoreInstrInfo.cpp b/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
index 075910c84fb84..6e2096c39f4e5 100644
--- a/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -377,7 +377,7 @@ void XCoreInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register DestReg, int FrameIndex,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL;
if (I != MBB.end() && !I->isDebugInstr())
diff --git a/llvm/lib/Target/XCore/XCoreInstrInfo.h b/llvm/lib/Target/XCore/XCoreInstrInfo.h
index c4e399ebd3fd8..ce69f786df57b 100644
--- a/llvm/lib/Target/XCore/XCoreInstrInfo.h
+++ b/llvm/lib/Target/XCore/XCoreInstrInfo.h
@@ -77,9 +77,8 @@ class XCoreInstrInfo : public XCoreGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
- int FrameIndex, const TargetRegisterClass *RC,
-
- Register VReg,
+ int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
bool reverseBranchCondition(
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
index d7b05acea9411..1eb42d1cd0c5d 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
@@ -159,7 +159,7 @@ void XtensaInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIdx,
const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg,
MachineInstr::MIFlag Flags) const {
DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
unsigned LoadOpcode, StoreOpcode;
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.h b/llvm/lib/Target/Xtensa/XtensaInstrInfo.h
index 0b46d6ce2fdb7..82e07de681116 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.h
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.h
@@ -62,7 +62,7 @@ class XtensaInstrInfo : public XtensaGenInstrInfo {
void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register DestReg, int FrameIdx, const TargetRegisterClass *RC,
- Register VReg,
+ Register VReg, unsigned SubReg = 0,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
// Get the load and store opcodes for a given register class and offset.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2fef934fa472e..c423b8a704b54 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -4954,10 +4954,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 28, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v49
; VI-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v60, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -4973,9 +4972,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 36, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5000,9 +4998,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5027,9 +5024,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 52, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5054,9 +5050,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 60, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5081,9 +5076,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5109,11 +5103,11 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5135,11 +5129,10 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5162,11 +5155,10 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5188,11 +5180,10 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x64, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v58
; VI-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -5214,11 +5205,10 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5241,11 +5231,10 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5977,9 +5966,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v61, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -5989,9 +5976,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6002,9 +5987,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6021,9 +6005,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6040,9 +6023,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6059,9 +6041,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6078,9 +6059,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6097,9 +6077,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -6125,12 +6104,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6151,12 +6130,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6177,12 +6156,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6203,12 +6182,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6229,12 +6208,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -6255,12 +6234,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -8181,15 +8160,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_readlane_b32 s18, v23, 0
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s19, v23, 1
; SI-NEXT: s_lshl_b32 s18, s18, 8
; SI-NEXT: s_or_b32 s16, s16, s18
; SI-NEXT: v_readlane_b32 s18, v23, 2
-; SI-NEXT: v_readlane_b32 s19, v23, 3
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: v_readlane_b32 s20, v23, 4
+; SI-NEXT: v_readlane_b32 s19, v23, 4
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s19, s20, 24
+; SI-NEXT: s_lshl_b32 s19, s19, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x44, v0
@@ -8215,15 +8192,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: v_readlane_b32 s16, v23, 6
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v23, 7
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v23, 8
-; SI-NEXT: v_readlane_b32 s17, v23, 9
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v23, 10
+; SI-NEXT: v_readlane_b32 s17, v23, 10
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x4c, v0
@@ -8249,15 +8224,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_readlane_b32 s14, v23, 12
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v23, 13
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v23, 14
-; SI-NEXT: v_readlane_b32 s15, v23, 15
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v23, 16
+; SI-NEXT: v_readlane_b32 s15, v23, 16
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x54, v0
@@ -8283,15 +8256,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: v_readlane_b32 s12, v23, 18
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v23, 19
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: v_readlane_b32 s12, v23, 20
-; SI-NEXT: v_readlane_b32 s13, v23, 21
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v23, 22
+; SI-NEXT: v_readlane_b32 s13, v23, 22
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x5c, v0
@@ -8317,15 +8288,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: v_readlane_b32 s10, v23, 24
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v23, 25
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
; SI-NEXT: v_readlane_b32 s10, v23, 26
-; SI-NEXT: v_readlane_b32 s11, v23, 27
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v23, 28
+; SI-NEXT: v_readlane_b32 s11, v23, 28
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x64, v0
@@ -8350,15 +8319,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_readlane_b32 s8, v23, 30
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v23, 31
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
; SI-NEXT: v_readlane_b32 s8, v23, 32
-; SI-NEXT: v_readlane_b32 s9, v23, 33
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v23, 34
+; SI-NEXT: v_readlane_b32 s9, v23, 34
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x6c, v0
@@ -8384,15 +8351,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_readlane_b32 s6, v23, 36
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s7, v23, 37
; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: v_readlane_b32 s6, v23, 38
-; SI-NEXT: v_readlane_b32 s7, v23, 39
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v23, 40
+; SI-NEXT: v_readlane_b32 s7, v23, 40
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x74, v0
@@ -8416,13 +8381,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s21, v23, 5
-; SI-NEXT: v_readlane_b32 s19, v23, 11
-; SI-NEXT: v_readlane_b32 s17, v23, 17
-; SI-NEXT: v_readlane_b32 s15, v23, 23
-; SI-NEXT: v_readlane_b32 s13, v23, 29
-; SI-NEXT: v_readlane_b32 s11, v23, 35
-; SI-NEXT: v_readlane_b32 s9, v23, 41
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s99, v21, 35
; SI-NEXT: v_readlane_b32 s98, v21, 34
@@ -9408,9 +9366,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_lshl_b32 s12, s12, 8
; VI-NEXT: s_or_b32 s10, s10, s12
; VI-NEXT: v_readlane_b32 s12, v22, 26
-; VI-NEXT: v_readlane_b32 s14, v22, 0
+; VI-NEXT: v_readlane_b32 s13, v22, 0
; VI-NEXT: s_and_b32 s12, s12, 0xff
-; VI-NEXT: s_lshl_b32 s13, s14, 8
+; VI-NEXT: s_lshl_b32 s13, s13, 8
; VI-NEXT: s_or_b32 s12, s12, s13
; VI-NEXT: s_and_b32 s10, s10, 0xffff
; VI-NEXT: s_lshl_b32 s12, s12, 16
@@ -9438,9 +9396,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_lshl_b32 s10, s10, 8
; VI-NEXT: s_or_b32 s8, s8, s10
; VI-NEXT: v_readlane_b32 s10, v22, 21
-; VI-NEXT: v_readlane_b32 s12, v22, 2
+; VI-NEXT: v_readlane_b32 s11, v22, 2
; VI-NEXT: s_and_b32 s10, s10, 0xff
-; VI-NEXT: s_lshl_b32 s11, s12, 8
+; VI-NEXT: s_lshl_b32 s11, s11, 8
; VI-NEXT: s_or_b32 s10, s10, s11
; VI-NEXT: s_and_b32 s8, s8, 0xffff
; VI-NEXT: s_lshl_b32 s10, s10, 16
@@ -9468,9 +9426,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_lshl_b32 s8, s8, 8
; VI-NEXT: s_or_b32 s6, s6, s8
; VI-NEXT: v_readlane_b32 s8, v22, 16
-; VI-NEXT: v_readlane_b32 s10, v22, 4
+; VI-NEXT: v_readlane_b32 s9, v22, 4
; VI-NEXT: s_and_b32 s8, s8, 0xff
-; VI-NEXT: s_lshl_b32 s9, s10, 8
+; VI-NEXT: s_lshl_b32 s9, s9, 8
; VI-NEXT: s_or_b32 s8, s8, s9
; VI-NEXT: s_and_b32 s6, s6, 0xffff
; VI-NEXT: s_lshl_b32 s8, s8, 16
@@ -9498,9 +9456,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_lshl_b32 s6, s6, 8
; VI-NEXT: s_or_b32 s4, s4, s6
; VI-NEXT: v_readlane_b32 s6, v22, 11
-; VI-NEXT: v_readlane_b32 s8, v22, 6
+; VI-NEXT: v_readlane_b32 s7, v22, 6
; VI-NEXT: s_and_b32 s6, s6, 0xff
-; VI-NEXT: s_lshl_b32 s7, s8, 8
+; VI-NEXT: s_lshl_b32 s7, s7, 8
; VI-NEXT: s_or_b32 s6, s6, s7
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s6, s6, 16
@@ -9524,10 +9482,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_readlane_b32 s15, v22, 1
-; VI-NEXT: v_readlane_b32 s13, v22, 3
-; VI-NEXT: v_readlane_b32 s11, v22, 5
-; VI-NEXT: v_readlane_b32 s9, v22, 7
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s87, v21, 31
; VI-NEXT: v_readlane_b32 s86, v21, 30
@@ -10527,9 +10481,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_lshl_b32 s6, s6, 8
; GFX9-NEXT: s_or_b32 s4, s4, s6
; GFX9-NEXT: v_readlane_b32 s6, v22, 5
-; GFX9-NEXT: v_readlane_b32 s8, v22, 0
+; GFX9-NEXT: v_readlane_b32 s7, v22, 0
; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshl_b32 s7, s8, 8
+; GFX9-NEXT: s_lshl_b32 s7, s7, 8
; GFX9-NEXT: s_or_b32 s6, s6, s7
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s6, s6, 16
@@ -10550,7 +10504,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_readlane_b32 s9, v22, 1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s99, v21, 35
; GFX9-NEXT: v_readlane_b32 s98, v21, 34
@@ -11515,7 +11468,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s16, s85, 8
; GFX11-NEXT: s_and_b32 s17, s84, 0xff
; GFX11-NEXT: s_lshl_b32 s18, s18, 8
-; GFX11-NEXT: v_readlane_b32 s19, v37, 1
; GFX11-NEXT: s_or_b32 s14, s14, s16
; GFX11-NEXT: s_or_b32 s16, s17, s18
; GFX11-NEXT: s_lshl_b32 s17, s83, 8
@@ -11533,7 +11485,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s16, s61, 8
; GFX11-NEXT: s_and_b32 s17, s80, 0xff
; GFX11-NEXT: s_lshl_b32 s18, s18, 8
-; GFX11-NEXT: v_readlane_b32 s19, v37, 3
; GFX11-NEXT: s_or_b32 s12, s12, s16
; GFX11-NEXT: s_or_b32 s16, s17, s18
; GFX11-NEXT: s_lshl_b32 s17, s60, 8
@@ -11554,7 +11505,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s12, s58, 8
; GFX11-NEXT: s_and_b32 s13, s59, 0xff
; GFX11-NEXT: s_lshl_b32 s14, s14, 8
-; GFX11-NEXT: v_readlane_b32 s15, v37, 5
; GFX11-NEXT: s_or_b32 s10, s10, s12
; GFX11-NEXT: s_or_b32 s12, s13, s14
; GFX11-NEXT: s_lshl_b32 s13, s68, 8
@@ -11572,7 +11522,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s12, s65, 8
; GFX11-NEXT: s_and_b32 s13, s64, 0xff
; GFX11-NEXT: s_lshl_b32 s14, s14, 8
-; GFX11-NEXT: v_readlane_b32 s15, v37, 7
; GFX11-NEXT: s_or_b32 s8, s8, s12
; GFX11-NEXT: s_or_b32 s12, s13, s14
; GFX11-NEXT: s_lshl_b32 s13, s55, 8
@@ -11596,7 +11545,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s8, s52, 8
; GFX11-NEXT: s_and_b32 s9, s51, 0xff
; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: v_readlane_b32 s11, v37, 9
; GFX11-NEXT: s_or_b32 s6, s6, s8
; GFX11-NEXT: s_or_b32 s8, s9, s10
; GFX11-NEXT: s_lshl_b32 s9, s50, 8
@@ -11614,7 +11562,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s8, s39, 8
; GFX11-NEXT: s_and_b32 s9, s38, 0xff
; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: v_readlane_b32 s11, v37, 11
; GFX11-NEXT: s_or_b32 s4, s4, s8
; GFX11-NEXT: s_or_b32 s8, s9, s10
; GFX11-NEXT: s_lshl_b32 s9, s37, 8
@@ -11635,7 +11582,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s4, s56, 8
; GFX11-NEXT: s_and_b32 s5, s57, 0xff
; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: v_readlane_b32 s7, v37, 13
; GFX11-NEXT: s_or_b32 s2, s2, s4
; GFX11-NEXT: s_or_b32 s4, s5, s6
; GFX11-NEXT: s_lshl_b32 s5, s34, 8
@@ -11653,7 +11599,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s4, s47, 8
; GFX11-NEXT: s_and_b32 s5, s104, 0xff
; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: v_readlane_b32 s7, v37, 15
; GFX11-NEXT: s_or_b32 s0, s0, s4
; GFX11-NEXT: s_or_b32 s4, s5, s6
; GFX11-NEXT: s_lshl_b32 s5, s103, 8
@@ -27786,19 +27731,18 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshr_b64 v[29:30], v[53:54], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v52, v53
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshr_b64 v[30:31], v[31:32], 16
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v57
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v53, v31
; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
; SI-NEXT: s_branch .LBB19_3
@@ -27864,76 +27808,62 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v42, v51
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB19_3: ; %Flow
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v56
; SI-NEXT: v_mov_b32_e32 v37, v34
; SI-NEXT: v_mov_b32_e32 v34, v33
-; SI-NEXT: v_mov_b32_e32 v35, v56
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v32, v40
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mov_b32_e32 v33, v38
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v51, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v54, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v44, v46
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v45, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v47, v56
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v58, v60
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mov_b32_e32 v32, v40
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_cbranch_vccnz .LBB19_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v61
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -27943,6 +27873,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
@@ -27952,15 +27883,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -27972,15 +27896,19 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
@@ -27988,37 +27916,55 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -28064,19 +28010,13 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -28088,78 +28028,42 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
@@ -28171,15 +28075,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
; SI-NEXT: .LBB19_5: ; %end
@@ -41485,10 +41383,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 28, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v49
; VI-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v60, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -41504,9 +41401,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 36, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41531,9 +41427,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41558,9 +41453,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 52, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41585,9 +41479,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 60, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41612,9 +41505,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41640,11 +41532,11 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41666,11 +41558,10 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -41693,11 +41584,10 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -41719,11 +41609,10 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x64, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v58
; VI-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -41745,11 +41634,10 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -41772,11 +41660,10 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -42508,9 +42395,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v61, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42520,9 +42405,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42533,9 +42416,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42552,9 +42434,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42571,9 +42452,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42590,9 +42470,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42609,9 +42488,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42628,9 +42506,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -42656,12 +42533,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42682,12 +42559,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42708,12 +42585,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42734,12 +42611,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42760,12 +42637,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42786,12 +42663,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -44377,8 +44254,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[16:17], s[16:17], 8
-; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[96:97], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[18:19], s[18:19], 8
; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
; SI-NEXT: s_lshr_b64 s[34:35], s[20:21], 16
@@ -44433,15 +44310,15 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v39, 24, v25
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
; SI-NEXT: v_lshrrev_b32_e32 v4, 8, v25
-; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v29
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v29
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v29
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v51, 8, v29
; SI-NEXT: v_lshrrev_b32_e32 v52, 24, v32
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v53, 8, v32
; SI-NEXT: v_lshrrev_b32_e32 v54, 24, v37
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v10, 8, v37
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v37
+; SI-NEXT: v_lshrrev_b32_e32 v55, 8, v37
; SI-NEXT: s_branch .LBB37_5
; SI-NEXT: .LBB37_3:
; SI-NEXT: ; implicit-def: $sgpr61
@@ -44628,17 +44505,17 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_readlane_b32 s5, v61, 42
-; SI-NEXT: v_mov_b32_e32 v10, s5
+; SI-NEXT: v_mov_b32_e32 v55, s5
; SI-NEXT: v_readlane_b32 s5, v61, 43
; SI-NEXT: v_mov_b32_e32 v53, s5
; SI-NEXT: v_readlane_b32 s5, v61, 44
; SI-NEXT: v_mov_b32_e32 v8, s5
; SI-NEXT: v_readlane_b32 s5, v61, 45
-; SI-NEXT: v_mov_b32_e32 v6, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 46
; SI-NEXT: v_mov_b32_e32 v51, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 47
+; SI-NEXT: v_readlane_b32 s5, v61, 46
; SI-NEXT: v_mov_b32_e32 v50, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 47
+; SI-NEXT: v_mov_b32_e32 v6, s5
; SI-NEXT: v_readlane_b32 s5, v61, 48
; SI-NEXT: v_mov_b32_e32 v4, s5
; SI-NEXT: v_readlane_b32 s5, v61, 49
@@ -44724,30 +44601,87 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s5, v62, 12
; SI-NEXT: v_mov_b32_e32 v57, s5
; SI-NEXT: v_readlane_b32 s5, v62, 13
-; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_mov_b32_e32 v56, s5
; SI-NEXT: v_readlane_b32 s5, v62, 14
+; SI-NEXT: v_mov_b32_e32 v46, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 15
+; SI-NEXT: v_mov_b32_e32 v45, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 16
+; SI-NEXT: v_mov_b32_e32 v1, s14
+; SI-NEXT: v_mov_b32_e32 v42, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 17
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s12
-; SI-NEXT: v_mov_b32_e32 v46, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 15
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 40
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s10
-; SI-NEXT: v_mov_b32_e32 v45, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 16
+; SI-NEXT: v_mov_b32_e32 v44, s26
+; SI-NEXT: s_mov_b32 s26, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 38
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: v_mov_b32_e32 v42, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 17
+; SI-NEXT: v_mov_b32_e32 v47, s28
+; SI-NEXT: s_mov_b32 s28, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 36
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b32 s6, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 34
+; SI-NEXT: s_mov_b32 s58, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 32
+; SI-NEXT: v_mov_b32_e32 v52, s60
+; SI-NEXT: s_mov_b32 s60, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 30
+; SI-NEXT: s_mov_b32 s8, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 28
+; SI-NEXT: s_mov_b32 s10, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 26
+; SI-NEXT: s_mov_b32 s12, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 24
+; SI-NEXT: s_mov_b32 s14, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 22
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v12, s16
+; SI-NEXT: s_mov_b32 s16, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 20
+; SI-NEXT: v_mov_b32_e32 v54, s96
+; SI-NEXT: s_mov_b32 s96, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 18
+; SI-NEXT: v_mov_b32_e32 v14, s18
+; SI-NEXT: s_mov_b32 s18, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 16
+; SI-NEXT: v_mov_b32_e32 v16, s20
+; SI-NEXT: s_mov_b32 s20, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 14
+; SI-NEXT: v_mov_b32_e32 v18, s22
+; SI-NEXT: s_mov_b32 s22, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 12
+; SI-NEXT: v_mov_b32_e32 v20, s24
+; SI-NEXT: s_mov_b32 s24, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 10
+; SI-NEXT: v_mov_b32_e32 v22, s40
+; SI-NEXT: s_mov_b32 s40, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 8
+; SI-NEXT: v_mov_b32_e32 v24, s42
+; SI-NEXT: s_mov_b32 s42, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 6
+; SI-NEXT: v_mov_b32_e32 v27, s44
+; SI-NEXT: s_mov_b32 s44, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 4
+; SI-NEXT: v_mov_b32_e32 v31, s46
+; SI-NEXT: s_mov_b32 s46, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 2
+; SI-NEXT: v_mov_b32_e32 v36, s56
+; SI-NEXT: s_mov_b32 s56, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 0
; SI-NEXT: v_mov_b32_e32 v37, s59
; SI-NEXT: v_mov_b32_e32 v32, s57
; SI-NEXT: v_mov_b32_e32 v29, s47
@@ -44766,69 +44700,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v54, s96
-; SI-NEXT: v_mov_b32_e32 v52, s60
-; SI-NEXT: v_mov_b32_e32 v47, s28
-; SI-NEXT: v_mov_b32_e32 v44, s26
+; SI-NEXT: v_mov_b32_e32 v10, s97
; SI-NEXT: v_mov_b32_e32 v43, s61
; SI-NEXT: v_mov_b32_e32 v41, s29
; SI-NEXT: v_mov_b32_e32 v40, s27
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s16
-; SI-NEXT: v_mov_b32_e32 v14, s18
-; SI-NEXT: v_mov_b32_e32 v16, s20
-; SI-NEXT: v_mov_b32_e32 v18, s22
-; SI-NEXT: v_mov_b32_e32 v20, s24
-; SI-NEXT: v_mov_b32_e32 v22, s40
-; SI-NEXT: v_mov_b32_e32 v24, s42
-; SI-NEXT: v_mov_b32_e32 v27, s44
-; SI-NEXT: v_mov_b32_e32 v31, s46
-; SI-NEXT: v_mov_b32_e32 v36, s56
-; SI-NEXT: v_readlane_b32 s26, v61, 40
-; SI-NEXT: v_readlane_b32 s28, v61, 38
-; SI-NEXT: v_readlane_b32 s6, v61, 36
-; SI-NEXT: v_readlane_b32 s58, v61, 34
-; SI-NEXT: v_readlane_b32 s60, v61, 32
-; SI-NEXT: v_readlane_b32 s8, v61, 30
-; SI-NEXT: v_readlane_b32 s10, v61, 28
-; SI-NEXT: v_readlane_b32 s12, v61, 26
-; SI-NEXT: v_readlane_b32 s14, v61, 24
-; SI-NEXT: v_readlane_b32 s16, v61, 22
-; SI-NEXT: s_mov_b32 s96, s94
-; SI-NEXT: v_readlane_b32 s94, v61, 20
-; SI-NEXT: v_readlane_b32 s18, v61, 18
-; SI-NEXT: v_readlane_b32 s20, v61, 16
-; SI-NEXT: v_readlane_b32 s22, v61, 14
-; SI-NEXT: v_readlane_b32 s24, v61, 12
-; SI-NEXT: v_readlane_b32 s40, v61, 10
-; SI-NEXT: v_readlane_b32 s42, v61, 8
-; SI-NEXT: v_readlane_b32 s44, v61, 6
-; SI-NEXT: v_readlane_b32 s46, v61, 4
-; SI-NEXT: v_readlane_b32 s56, v61, 2
-; SI-NEXT: v_readlane_b32 vcc_lo, v61, 0
-; SI-NEXT: v_mov_b32_e32 v55, s97
-; SI-NEXT: v_readlane_b32 s27, v61, 41
-; SI-NEXT: v_readlane_b32 s29, v61, 39
-; SI-NEXT: v_readlane_b32 s7, v61, 37
-; SI-NEXT: v_readlane_b32 s59, v61, 35
-; SI-NEXT: v_readlane_b32 s61, v61, 33
-; SI-NEXT: v_readlane_b32 s9, v61, 31
-; SI-NEXT: v_readlane_b32 s11, v61, 29
-; SI-NEXT: v_readlane_b32 s13, v61, 27
-; SI-NEXT: v_readlane_b32 s15, v61, 25
-; SI-NEXT: v_readlane_b32 s17, v61, 23
-; SI-NEXT: v_readlane_b32 s95, v61, 21
-; SI-NEXT: v_readlane_b32 s19, v61, 19
-; SI-NEXT: v_readlane_b32 s21, v61, 17
-; SI-NEXT: v_readlane_b32 s23, v61, 15
-; SI-NEXT: v_readlane_b32 s25, v61, 13
-; SI-NEXT: v_readlane_b32 s41, v61, 11
-; SI-NEXT: v_readlane_b32 s43, v61, 9
-; SI-NEXT: v_readlane_b32 s45, v61, 7
-; SI-NEXT: v_readlane_b32 s47, v61, 5
-; SI-NEXT: v_readlane_b32 s57, v61, 3
-; SI-NEXT: v_readlane_b32 vcc_hi, v61, 1
+; SI-NEXT: s_mov_b32 vcc_lo, s5
; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v48, 0xff, v48
@@ -44839,18 +44715,18 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshl_b32 s5, vcc_lo, 24
; SI-NEXT: v_and_b32_e32 v48, 0xffff, v48
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_and_b32_e32 v37, 0xff, v37
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
; SI-NEXT: v_or_b32_e32 v48, s4, v48
-; SI-NEXT: v_or_b32_e32 v10, v37, v10
-; SI-NEXT: v_and_b32_e32 v37, 0xff, v55
; SI-NEXT: buffer_store_dword v48, v0, s[0:3], 0 offen
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v37, 0xff, v37
; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v48, 8, v55
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT: v_or_b32_e32 v37, v37, v48
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshlrev_b32_e32 v48, 24, v54
-; SI-NEXT: v_or_b32_e32 v37, v48, v37
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_or_b32_e32 v10, v10, v37
+; SI-NEXT: v_or_b32_e32 v10, v48, v10
+; SI-NEXT: v_and_b32_e32 v37, 0xffff, v37
+; SI-NEXT: v_or_b32_e32 v10, v37, v10
; SI-NEXT: v_add_i32_e32 v37, vcc, 4, v0
; SI-NEXT: buffer_store_dword v10, v37, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
@@ -44891,14 +44767,14 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v8, v10, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xff, v29
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
+; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v51
+; SI-NEXT: v_or_b32_e32 v8, v8, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v50
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_or_b32_e32 v6, v6, v10
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v51
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v50
-; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
; SI-NEXT: v_add_i32_e32 v8, vcc, 20, v0
; SI-NEXT: buffer_store_dword v6, v8, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
@@ -45093,7 +44969,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v15
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s18, 8
-; SI-NEXT: s_lshl_b32 s5, s94, 24
+; SI-NEXT: s_lshl_b32 s5, s96, 24
+; SI-NEXT: v_readlane_b32 s96, v63, 32
; SI-NEXT: v_readlane_b32 s34, v63, 2
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
@@ -45112,7 +44989,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v14
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s96, 0xff
+; SI-NEXT: s_and_b32 s4, s94, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -45125,7 +45002,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v34
; SI-NEXT: s_lshl_b32 s4, s16, 8
; SI-NEXT: s_lshl_b32 s5, s90, 24
-; SI-NEXT: v_readlane_b32 s96, v63, 32
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -46303,9 +46179,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v24, vcc, 36, v0
; VI-NEXT: buffer_store_dword v23, v24, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v23, 8, v23
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -46329,9 +46204,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v22, vcc, 44, v0
; VI-NEXT: buffer_store_dword v21, v22, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v21, 8, v21
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -46355,9 +46229,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v20, vcc, 52, v0
; VI-NEXT: buffer_store_dword v19, v20, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v19, 8, v19
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -46381,9 +46254,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v18, vcc, 60, v0
; VI-NEXT: buffer_store_dword v17, v18, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v17, 8, v17
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -46407,9 +46279,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v16, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v15, v16, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -46433,15 +46304,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x4c, v0
; VI-NEXT: buffer_store_dword v13, v14, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; VI-NEXT: v_or_b32_sdwa v11, v11, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; VI-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v11, v11, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x50, v0
@@ -46459,15 +46328,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v12, vcc, 0x54, v0
; VI-NEXT: buffer_store_dword v11, v12, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; VI-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; VI-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v11, vcc, 0x58, v0
@@ -46485,15 +46352,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v10, vcc, 0x5c, v0
; VI-NEXT: buffer_store_dword v9, v10, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; VI-NEXT: v_or_b32_sdwa v7, v7, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; VI-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v7, v7, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x60, v0
@@ -46511,15 +46376,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x64, v0
; VI-NEXT: buffer_store_dword v7, v8, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; VI-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; VI-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x68, v0
@@ -46537,15 +46400,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x6c, v0
; VI-NEXT: buffer_store_dword v5, v6, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; VI-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x70, v0
@@ -46563,15 +46424,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_u32_e32 v4, vcc, 0x74, v0
; VI-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; VI-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x78, v0
@@ -47608,9 +47467,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47627,9 +47485,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47646,9 +47503,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47665,9 +47521,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47684,9 +47539,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47703,9 +47557,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47722,9 +47575,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47751,9 +47603,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: v_or_b32_sdwa v11, v11, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47776,9 +47627,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47801,9 +47651,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: v_or_b32_sdwa v7, v7, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47826,9 +47675,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47851,9 +47699,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -47876,9 +47723,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -65079,19 +64925,18 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshr_b64 v[29:30], v[53:54], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v52, v53
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshr_b64 v[30:31], v[31:32], 16
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v57
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v53, v31
; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
; SI-NEXT: s_branch .LBB43_3
@@ -65157,76 +65002,62 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v42, v51
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB43_3: ; %Flow
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v56
; SI-NEXT: v_mov_b32_e32 v37, v34
; SI-NEXT: v_mov_b32_e32 v34, v33
-; SI-NEXT: v_mov_b32_e32 v35, v56
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v32, v40
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mov_b32_e32 v33, v38
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v51, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v54, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v44, v46
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v45, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v47, v56
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v58, v60
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mov_b32_e32 v32, v40
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_cbranch_vccnz .LBB43_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v61
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -65236,6 +65067,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
@@ -65245,15 +65077,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -65265,15 +65090,19 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
@@ -65281,37 +65110,55 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -65357,19 +65204,13 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -65381,78 +65222,42 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
@@ -65464,15 +65269,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
; SI-NEXT: .LBB43_5: ; %end
@@ -72992,10 +72791,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x64, v0
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x68, v0
@@ -73007,10 +72805,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x6c, v0
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x70, v0
@@ -73022,10 +72819,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x74, v0
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x78, v0
@@ -77814,10 +77610,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 28, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v49
; VI-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v60, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -77833,9 +77628,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 36, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77860,9 +77654,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77887,9 +77680,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 52, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77914,9 +77706,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 60, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77941,9 +77732,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77969,11 +77759,11 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -77995,11 +77785,10 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -78022,11 +77811,10 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -78048,11 +77836,10 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x64, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v58
; VI-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78074,11 +77861,10 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -78101,11 +77887,10 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -78837,9 +78622,7 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v61, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -78849,9 +78632,7 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -78862,9 +78643,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78881,9 +78661,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78900,9 +78679,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78919,9 +78697,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78938,9 +78715,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78957,9 +78733,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -78985,12 +78760,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -79011,12 +78786,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -79037,12 +78812,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -79063,12 +78838,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -79089,12 +78864,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -79115,12 +78890,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -81058,15 +80833,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_readlane_b32 s18, v23, 0
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s19, v23, 1
; SI-NEXT: s_lshl_b32 s18, s18, 8
; SI-NEXT: s_or_b32 s16, s16, s18
; SI-NEXT: v_readlane_b32 s18, v23, 2
-; SI-NEXT: v_readlane_b32 s19, v23, 3
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: v_readlane_b32 s20, v23, 4
+; SI-NEXT: v_readlane_b32 s19, v23, 4
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s19, s20, 24
+; SI-NEXT: s_lshl_b32 s19, s19, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x44, v0
@@ -81092,15 +80865,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: v_readlane_b32 s16, v23, 6
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v23, 7
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v23, 8
-; SI-NEXT: v_readlane_b32 s17, v23, 9
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v23, 10
+; SI-NEXT: v_readlane_b32 s17, v23, 10
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x4c, v0
@@ -81126,15 +80897,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_readlane_b32 s14, v23, 12
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v23, 13
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v23, 14
-; SI-NEXT: v_readlane_b32 s15, v23, 15
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v23, 16
+; SI-NEXT: v_readlane_b32 s15, v23, 16
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x54, v0
@@ -81160,15 +80929,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: v_readlane_b32 s12, v23, 18
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v23, 19
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: v_readlane_b32 s12, v23, 20
-; SI-NEXT: v_readlane_b32 s13, v23, 21
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v23, 22
+; SI-NEXT: v_readlane_b32 s13, v23, 22
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x5c, v0
@@ -81194,15 +80961,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: v_readlane_b32 s10, v23, 24
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v23, 25
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
; SI-NEXT: v_readlane_b32 s10, v23, 26
-; SI-NEXT: v_readlane_b32 s11, v23, 27
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v23, 28
+; SI-NEXT: v_readlane_b32 s11, v23, 28
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x64, v0
@@ -81228,15 +80993,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_readlane_b32 s8, v23, 30
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v23, 31
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
; SI-NEXT: v_readlane_b32 s8, v23, 32
-; SI-NEXT: v_readlane_b32 s9, v23, 33
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v23, 34
+; SI-NEXT: v_readlane_b32 s9, v23, 34
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x6c, v0
@@ -81263,12 +81026,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readlane_b32 s6, v23, 36
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: v_readlane_b32 s7, v23, 37
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s54, 0xff
-; SI-NEXT: v_readlane_b32 s8, v23, 38
+; SI-NEXT: v_readlane_b32 s7, v23, 38
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x74, v0
@@ -81292,13 +81054,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s21, v23, 5
-; SI-NEXT: v_readlane_b32 s19, v23, 11
-; SI-NEXT: v_readlane_b32 s17, v23, 17
-; SI-NEXT: v_readlane_b32 s15, v23, 23
-; SI-NEXT: v_readlane_b32 s13, v23, 29
-; SI-NEXT: v_readlane_b32 s11, v23, 35
-; SI-NEXT: v_readlane_b32 s9, v23, 39
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s99, v21, 35
; SI-NEXT: v_readlane_b32 s98, v21, 34
@@ -82284,9 +82039,9 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_lshl_b32 s12, s12, 8
; VI-NEXT: s_or_b32 s10, s10, s12
; VI-NEXT: v_readlane_b32 s12, v22, 26
-; VI-NEXT: v_readlane_b32 s14, v22, 0
+; VI-NEXT: v_readlane_b32 s13, v22, 0
; VI-NEXT: s_and_b32 s12, s12, 0xff
-; VI-NEXT: s_lshl_b32 s13, s14, 8
+; VI-NEXT: s_lshl_b32 s13, s13, 8
; VI-NEXT: s_or_b32 s12, s12, s13
; VI-NEXT: s_and_b32 s10, s10, 0xffff
; VI-NEXT: s_lshl_b32 s12, s12, 16
@@ -82314,9 +82069,9 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_lshl_b32 s10, s10, 8
; VI-NEXT: s_or_b32 s8, s8, s10
; VI-NEXT: v_readlane_b32 s10, v22, 21
-; VI-NEXT: v_readlane_b32 s12, v22, 2
+; VI-NEXT: v_readlane_b32 s11, v22, 2
; VI-NEXT: s_and_b32 s10, s10, 0xff
-; VI-NEXT: s_lshl_b32 s11, s12, 8
+; VI-NEXT: s_lshl_b32 s11, s11, 8
; VI-NEXT: s_or_b32 s10, s10, s11
; VI-NEXT: s_and_b32 s8, s8, 0xffff
; VI-NEXT: s_lshl_b32 s10, s10, 16
@@ -82344,9 +82099,9 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_lshl_b32 s8, s8, 8
; VI-NEXT: s_or_b32 s6, s6, s8
; VI-NEXT: v_readlane_b32 s8, v22, 16
-; VI-NEXT: v_readlane_b32 s10, v22, 4
+; VI-NEXT: v_readlane_b32 s9, v22, 4
; VI-NEXT: s_and_b32 s8, s8, 0xff
-; VI-NEXT: s_lshl_b32 s9, s10, 8
+; VI-NEXT: s_lshl_b32 s9, s9, 8
; VI-NEXT: s_or_b32 s8, s8, s9
; VI-NEXT: s_and_b32 s6, s6, 0xffff
; VI-NEXT: s_lshl_b32 s8, s8, 16
@@ -82374,9 +82129,9 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_lshl_b32 s6, s6, 8
; VI-NEXT: s_or_b32 s4, s4, s6
; VI-NEXT: v_readlane_b32 s6, v22, 11
-; VI-NEXT: v_readlane_b32 s8, v22, 6
+; VI-NEXT: v_readlane_b32 s7, v22, 6
; VI-NEXT: s_and_b32 s6, s6, 0xff
-; VI-NEXT: s_lshl_b32 s7, s8, 8
+; VI-NEXT: s_lshl_b32 s7, s7, 8
; VI-NEXT: s_or_b32 s6, s6, s7
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s6, s6, 16
@@ -82400,10 +82155,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_readlane_b32 s15, v22, 1
-; VI-NEXT: v_readlane_b32 s13, v22, 3
-; VI-NEXT: v_readlane_b32 s11, v22, 5
-; VI-NEXT: v_readlane_b32 s9, v22, 7
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s87, v21, 31
; VI-NEXT: v_readlane_b32 s86, v21, 30
@@ -83403,9 +83154,9 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_lshl_b32 s6, s6, 8
; GFX9-NEXT: s_or_b32 s4, s4, s6
; GFX9-NEXT: v_readlane_b32 s6, v22, 5
-; GFX9-NEXT: v_readlane_b32 s8, v22, 0
+; GFX9-NEXT: v_readlane_b32 s7, v22, 0
; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshl_b32 s7, s8, 8
+; GFX9-NEXT: s_lshl_b32 s7, s7, 8
; GFX9-NEXT: s_or_b32 s6, s6, s7
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s6, s6, 16
@@ -83426,7 +83177,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_readlane_b32 s9, v22, 1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s99, v21, 35
; GFX9-NEXT: v_readlane_b32 s98, v21, 34
@@ -84393,7 +84143,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s16, s85, 8
; GFX11-NEXT: s_and_b32 s17, s84, 0xff
; GFX11-NEXT: s_lshl_b32 s18, s18, 8
-; GFX11-NEXT: v_readlane_b32 s19, v37, 1
; GFX11-NEXT: s_or_b32 s14, s14, s16
; GFX11-NEXT: s_or_b32 s16, s17, s18
; GFX11-NEXT: s_lshl_b32 s17, s83, 8
@@ -84411,7 +84160,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s16, s61, 8
; GFX11-NEXT: s_and_b32 s17, s80, 0xff
; GFX11-NEXT: s_lshl_b32 s18, s18, 8
-; GFX11-NEXT: v_readlane_b32 s19, v37, 3
; GFX11-NEXT: s_or_b32 s12, s12, s16
; GFX11-NEXT: s_or_b32 s16, s17, s18
; GFX11-NEXT: s_lshl_b32 s17, s60, 8
@@ -84432,7 +84180,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s12, s58, 8
; GFX11-NEXT: s_and_b32 s13, s59, 0xff
; GFX11-NEXT: s_lshl_b32 s14, s14, 8
-; GFX11-NEXT: v_readlane_b32 s15, v37, 5
; GFX11-NEXT: s_or_b32 s10, s10, s12
; GFX11-NEXT: s_or_b32 s12, s13, s14
; GFX11-NEXT: s_lshl_b32 s13, s68, 8
@@ -84450,7 +84197,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s12, s65, 8
; GFX11-NEXT: s_and_b32 s13, s64, 0xff
; GFX11-NEXT: s_lshl_b32 s14, s14, 8
-; GFX11-NEXT: v_readlane_b32 s15, v37, 7
; GFX11-NEXT: s_or_b32 s8, s8, s12
; GFX11-NEXT: s_or_b32 s12, s13, s14
; GFX11-NEXT: s_lshl_b32 s13, s55, 8
@@ -84474,7 +84220,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s8, s52, 8
; GFX11-NEXT: s_and_b32 s9, s51, 0xff
; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: v_readlane_b32 s11, v37, 9
; GFX11-NEXT: s_or_b32 s6, s6, s8
; GFX11-NEXT: s_or_b32 s8, s9, s10
; GFX11-NEXT: s_lshl_b32 s9, s50, 8
@@ -84492,7 +84237,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s8, s39, 8
; GFX11-NEXT: s_and_b32 s9, s38, 0xff
; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: v_readlane_b32 s11, v37, 11
; GFX11-NEXT: s_or_b32 s4, s4, s8
; GFX11-NEXT: s_or_b32 s8, s9, s10
; GFX11-NEXT: s_lshl_b32 s9, s37, 8
@@ -84513,7 +84257,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s4, s56, 8
; GFX11-NEXT: s_and_b32 s5, s57, 0xff
; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: v_readlane_b32 s7, v37, 13
; GFX11-NEXT: s_or_b32 s2, s2, s4
; GFX11-NEXT: s_or_b32 s4, s5, s6
; GFX11-NEXT: s_lshl_b32 s5, s34, 8
@@ -84531,7 +84274,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_lshl_b32 s4, s47, 8
; GFX11-NEXT: s_and_b32 s5, s104, 0xff
; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: v_readlane_b32 s7, v37, 15
; GFX11-NEXT: s_or_b32 s0, s0, s4
; GFX11-NEXT: s_or_b32 s4, s5, s6
; GFX11-NEXT: s_lshl_b32 s5, s103, 8
@@ -100643,19 +100385,18 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshr_b64 v[29:30], v[53:54], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v52, v53
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshr_b64 v[30:31], v[31:32], 16
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v57
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v53, v31
; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
; SI-NEXT: s_branch .LBB63_3
@@ -100721,76 +100462,62 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v42, v51
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB63_3: ; %Flow
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v56
; SI-NEXT: v_mov_b32_e32 v37, v34
; SI-NEXT: v_mov_b32_e32 v34, v33
-; SI-NEXT: v_mov_b32_e32 v35, v56
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v32, v40
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mov_b32_e32 v33, v38
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v51, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v54, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v44, v46
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v45, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v47, v56
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v58, v60
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mov_b32_e32 v32, v40
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_cbranch_vccnz .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v61
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -100800,6 +100527,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
@@ -100809,15 +100537,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -100829,15 +100550,19 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
@@ -100845,37 +100570,55 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -100921,19 +100664,13 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -100945,78 +100682,42 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
@@ -101028,15 +100729,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
; SI-NEXT: .LBB63_5: ; %end
@@ -112342,10 +112037,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 20, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v37
; VI-NEXT: v_or_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v61, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -112359,10 +112053,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 28, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v59
; VI-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v36, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -112376,9 +112069,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 36, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112401,9 +112093,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112428,9 +112119,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 52, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112455,9 +112145,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 60, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112482,9 +112171,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112510,11 +112198,11 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -112536,11 +112224,10 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112563,11 +112250,10 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112590,11 +112276,10 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112615,11 +112300,10 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112642,11 +112326,10 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -113362,9 +113045,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v62, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113374,9 +113055,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v51, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113386,9 +113065,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v49, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113398,9 +113075,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v47, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113416,9 +113091,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -113435,9 +113109,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -113454,9 +113127,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -113473,9 +113145,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -113492,9 +113163,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -113520,12 +113190,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113546,12 +113216,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113572,12 +113242,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113598,12 +113268,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113624,12 +113294,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -113650,12 +113320,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -115106,176 +114776,164 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[38:39], s[56:57], 8
; SI-NEXT: s_cbranch_execnz .LBB73_4
; SI-NEXT: .LBB73_2: ; %cmp.true
-; SI-NEXT: v_add_f64 v[30:31], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[7:8], s[24:25], 1.0
-; SI-NEXT: v_lshr_b64 v[9:10], v[30:31], 24
-; SI-NEXT: v_lshr_b64 v[10:11], v[7:8], 24
-; SI-NEXT: v_add_f64 v[54:55], s[40:41], 1.0
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f64 v[1:2], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[3:4], s[6:7], 1.0
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 24
+; SI-NEXT: v_add_f64 v[5:6], s[8:9], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 16
+; SI-NEXT: v_add_f64 v[7:8], s[10:11], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 8
+; SI-NEXT: v_add_f64 v[9:10], s[12:13], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[54:55], 24
-; SI-NEXT: v_add_f64 v[42:43], s[42:43], 1.0
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 24
+; SI-NEXT: v_add_f64 v[11:12], s[14:15], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 16
+; SI-NEXT: v_add_f64 v[13:14], s[16:17], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[54:55], 16
-; SI-NEXT: v_add_f64 v[37:38], s[4:5], 1.0
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 8
+; SI-NEXT: v_readfirstlane_b32 s17, v14
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 24
+; SI-NEXT: v_add_f64 v[18:19], s[18:19], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 16
+; SI-NEXT: v_add_f64 v[25:26], s[20:21], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 8
+; SI-NEXT: v_add_f64 v[31:32], s[22:23], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 24
+; SI-NEXT: v_add_f64 v[37:38], s[24:25], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 16
+; SI-NEXT: v_add_f64 v[43:44], s[42:43], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 8
+; SI-NEXT: v_lshr_b64 v[47:48], v[31:32], 16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[42:43], 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 24
; SI-NEXT: v_add_f64 v[56:57], s[44:45], 1.0
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[42:43], 8
-; SI-NEXT: v_lshr_b64 v[1:2], v[37:38], 24
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 16
+; SI-NEXT: v_lshr_b64 v[33:34], v[31:32], 24
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[56:57], 24
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[37:38], 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 8
+; SI-NEXT: v_lshr_b64 v[48:49], v[31:32], 8
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[56:57], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[37:38], 8
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 24
+; SI-NEXT: v_add_f64 v[27:28], s[56:57], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[56:57], 8
-; SI-NEXT: v_add_f64 v[35:36], s[6:7], 1.0
-; SI-NEXT: v_add_f64 v[19:20], s[46:47], 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[35:36], 24
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 16
+; SI-NEXT: v_add_f64 v[20:21], s[46:47], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[19:20], 24
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 8
+; SI-NEXT: v_lshr_b64 v[34:35], v[37:38], 8
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[19:20], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[35:36], 8
+; SI-NEXT: v_lshr_b64 v[15:16], v[13:14], 24
+; SI-NEXT: v_lshr_b64 v[49:50], v[43:44], 16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[19:20], 8
-; SI-NEXT: v_add_f64 v[3:4], s[8:9], 1.0
-; SI-NEXT: v_add_f64 v[24:25], s[56:57], 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 24
+; SI-NEXT: v_lshr_b64 v[15:16], v[13:14], 16
+; SI-NEXT: v_readfirstlane_b32 s43, v44
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshr_b64 v[14:15], v[13:14], 8
+; SI-NEXT: v_lshr_b64 v[40:41], v[37:38], 24
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[24:25], 24
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 24
+; SI-NEXT: v_lshr_b64 v[35:36], v[43:44], 24
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[24:25], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 8
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[43:44], 8
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[24:25], 8
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 8
+; SI-NEXT: v_lshr_b64 v[44:45], v[56:57], 16
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f64 v[47:48], s[10:11], 1.0
-; SI-NEXT: v_add_f64 v[13:14], s[16:17], 1.0
-; SI-NEXT: v_lshr_b64 v[1:2], v[47:48], 24
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f64 v[32:33], s[14:15], 1.0
-; SI-NEXT: v_add_f64 v[28:29], s[18:19], 1.0
+; SI-NEXT: v_lshr_b64 v[14:15], v[25:26], 24
+; SI-NEXT: v_add_f64 v[52:53], s[40:41], 1.0
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[47:48], 8
-; SI-NEXT: v_lshr_b64 v[17:18], v[13:14], 16
-; SI-NEXT: v_add_f64 v[21:22], s[20:21], 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[15:16], v[32:33], 8
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[26:27], v[28:29], 8
-; SI-NEXT: v_readfirstlane_b32 s11, v48
-; SI-NEXT: v_lshr_b64 v[51:52], v[47:48], 16
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[1:2], v[28:29], 24
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[16:17], v[28:29], 16
-; SI-NEXT: v_mov_b32_e32 v48, v28
-; SI-NEXT: v_lshr_b64 v[27:28], v[21:22], 8
-; SI-NEXT: v_readfirstlane_b32 s19, v29
-; SI-NEXT: v_lshr_b64 v[28:29], v[30:31], 16
-; SI-NEXT: v_readfirstlane_b32 s23, v31
-; SI-NEXT: v_lshr_b64 v[52:53], v[30:31], 8
-; SI-NEXT: v_readfirstlane_b32 s47, v20
-; SI-NEXT: v_readfirstlane_b32 s43, v43
-; SI-NEXT: v_readfirstlane_b32 s5, v38
-; SI-NEXT: v_lshr_b64 v[38:39], v[42:43], 24
-; SI-NEXT: v_add_f64 v[44:45], s[12:13], 1.0
-; SI-NEXT: v_readfirstlane_b32 s9, v4
-; SI-NEXT: v_lshr_b64 v[58:59], v[44:45], 24
-; SI-NEXT: v_lshr_b64 v[4:5], v[32:33], 16
-; SI-NEXT: v_readfirstlane_b32 s15, v33
-; SI-NEXT: v_readfirstlane_b32 s13, v45
-; SI-NEXT: v_readfirstlane_b32 s7, v36
-; SI-NEXT: v_lshr_b64 v[40:41], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[59:60], v[44:45], 8
-; SI-NEXT: v_lshr_b64 v[45:46], v[32:33], 24
-; SI-NEXT: v_lshr_b64 v[33:34], v[13:14], 24
-; SI-NEXT: v_lshr_b64 v[5:6], v[13:14], 8
-; SI-NEXT: v_mov_b32_e32 v36, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[21:22], 24
-; SI-NEXT: v_readfirstlane_b32 s17, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[7:8], 16
-; SI-NEXT: v_lshr_b64 v[49:50], v[54:55], 8
-; SI-NEXT: v_mov_b32_e32 v14, v9
-; SI-NEXT: v_readfirstlane_b32 s57, v25
-; SI-NEXT: v_mov_b32_e32 v25, v12
-; SI-NEXT: v_mov_b32_e32 v12, v5
-; SI-NEXT: v_mov_b32_e32 v5, v16
-; SI-NEXT: v_readfirstlane_b32 s25, v8
-; SI-NEXT: v_readfirstlane_b32 s21, v22
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[21:22], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[7:8], 8
-; SI-NEXT: v_mov_b32_e32 v23, v27
-; SI-NEXT: v_mov_b32_e32 v27, v17
-; SI-NEXT: v_mov_b32_e32 v17, v15
+; SI-NEXT: v_lshr_b64 v[14:15], v[25:26], 16
; SI-NEXT: v_readfirstlane_b32 s45, v57
-; SI-NEXT: v_readfirstlane_b32 s41, v55
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[41:42], v[56:57], 24
+; SI-NEXT: v_lshr_b64 v[45:46], v[56:57], 8
+; SI-NEXT: v_lshr_b64 v[57:58], v[20:21], 24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[27:28], 24
+; SI-NEXT: v_lshr_b64 v[22:23], v[25:26], 8
+; SI-NEXT: v_lshr_b64 v[58:59], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[27:28], 16
+; SI-NEXT: v_readfirstlane_b32 s57, v28
+; SI-NEXT: v_readfirstlane_b32 s47, v21
+; SI-NEXT: v_readfirstlane_b32 s41, v53
+; SI-NEXT: v_readfirstlane_b32 s25, v38
+; SI-NEXT: v_readfirstlane_b32 s23, v32
+; SI-NEXT: v_readfirstlane_b32 s21, v26
+; SI-NEXT: v_readfirstlane_b32 s19, v19
+; SI-NEXT: v_readfirstlane_b32 s15, v12
+; SI-NEXT: v_readfirstlane_b32 s13, v10
+; SI-NEXT: v_readfirstlane_b32 s11, v8
+; SI-NEXT: v_readfirstlane_b32 s9, v6
+; SI-NEXT: v_readfirstlane_b32 s7, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v2
+; SI-NEXT: v_lshr_b64 v[29:30], v[37:38], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[52:53], 24
+; SI-NEXT: v_lshr_b64 v[38:39], v[52:53], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[52:53], 8
+; SI-NEXT: v_lshr_b64 v[59:60], v[20:21], 8
+; SI-NEXT: v_lshr_b64 v[16:17], v[27:28], 8
; SI-NEXT: s_lshr_b32 s10, s5, 24
; SI-NEXT: s_lshr_b32 s12, s5, 16
; SI-NEXT: s_lshr_b32 s14, s5, 8
@@ -115295,10 +114953,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b32 s56, s15, 16
; SI-NEXT: s_lshr_b32 s58, s15, 8
; SI-NEXT: s_lshr_b32 s59, s17, 24
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mov_b32_e32 v29, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s60, s17, 16
; SI-NEXT: s_lshr_b32 s61, s17, 8
; SI-NEXT: s_lshr_b32 s62, s19, 24
@@ -115328,44 +114982,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b32 s8, s57, 24
; SI-NEXT: s_lshr_b32 vcc_lo, s57, 16
; SI-NEXT: s_lshr_b32 s6, s57, 8
-; SI-NEXT: v_mov_b32_e32 v34, v4
-; SI-NEXT: v_mov_b32_e32 v53, v51
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v31, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v20, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v43, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v10
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v46, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v16, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v8, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v15, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v51, v10
; SI-NEXT: s_branch .LBB73_5
; SI-NEXT: .LBB73_3:
; SI-NEXT: ; implicit-def: $sgpr26
@@ -115548,97 +115164,50 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: ; implicit-def: $sgpr48
; SI-NEXT: s_branch .LBB73_2
; SI-NEXT: .LBB73_4:
-; SI-NEXT: v_mov_b32_e32 v1, s38
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v2, s80
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s36
-; SI-NEXT: v_mov_b32_e32 v13, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 0
-; SI-NEXT: v_readlane_b32 s27, v62, 1
-; SI-NEXT: v_mov_b32_e32 v17, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 2
-; SI-NEXT: v_readlane_b32 s27, v62, 3
-; SI-NEXT: v_mov_b32_e32 v34, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 4
-; SI-NEXT: v_readlane_b32 s27, v62, 5
-; SI-NEXT: v_mov_b32_e32 v45, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 6
-; SI-NEXT: v_readlane_b32 s27, v62, 7
-; SI-NEXT: v_mov_b32_e32 v59, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 8
-; SI-NEXT: v_readlane_b32 s27, v62, 9
-; SI-NEXT: v_mov_b32_e32 v40, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 10
-; SI-NEXT: v_readlane_b32 s27, v62, 11
-; SI-NEXT: v_mov_b32_e32 v58, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 12
-; SI-NEXT: v_readlane_b32 s27, v62, 13
-; SI-NEXT: v_mov_b32_e32 v9, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 14
-; SI-NEXT: v_readlane_b32 s27, v62, 15
-; SI-NEXT: v_mov_b32_e32 v53, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 16
-; SI-NEXT: v_readlane_b32 s27, v62, 17
-; SI-NEXT: v_mov_b32_e32 v50, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 18
-; SI-NEXT: v_readlane_b32 s27, v62, 19
+; SI-NEXT: v_mov_b32_e32 v2, s70
+; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: v_mov_b32_e32 v16, s38
+; SI-NEXT: v_mov_b32_e32 v15, s36
+; SI-NEXT: v_mov_b32_e32 v14, s34
+; SI-NEXT: v_mov_b32_e32 v59, s30
+; SI-NEXT: v_mov_b32_e32 v58, s94
+; SI-NEXT: v_mov_b32_e32 v57, s92
+; SI-NEXT: v_mov_b32_e32 v45, s90
+; SI-NEXT: v_mov_b32_e32 v44, s88
+; SI-NEXT: v_mov_b32_e32 v41, s78
+; SI-NEXT: v_mov_b32_e32 v50, s76
+; SI-NEXT: v_mov_b32_e32 v49, s74
+; SI-NEXT: v_mov_b32_e32 v35, s72
+; SI-NEXT: v_mov_b32_e32 v23, s62
+; SI-NEXT: v_mov_b32_e32 v38, s60
+; SI-NEXT: v_mov_b32_e32 v54, s58
+; SI-NEXT: v_mov_b32_e32 v34, s28
; SI-NEXT: v_mov_b32_e32 v29, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 20
-; SI-NEXT: v_readlane_b32 s27, v62, 21
-; SI-NEXT: v_mov_b32_e32 v31, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 22
-; SI-NEXT: v_readlane_b32 s27, v62, 23
-; SI-NEXT: v_mov_b32_e32 v20, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 24
-; SI-NEXT: v_readlane_b32 s27, v62, 25
-; SI-NEXT: v_mov_b32_e32 v43, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 26
-; SI-NEXT: v_readlane_b32 s27, v62, 27
-; SI-NEXT: v_mov_b32_e32 v46, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 28
-; SI-NEXT: v_readlane_b32 s27, v62, 29
-; SI-NEXT: v_mov_b32_e32 v16, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 30
-; SI-NEXT: v_readlane_b32 s27, v62, 31
-; SI-NEXT: v_mov_b32_e32 v51, s26
-; SI-NEXT: v_readlane_b32 s26, v62, 32
-; SI-NEXT: v_mov_b32_e32 v3, s8
-; SI-NEXT: v_readlane_b32 s27, v62, 33
-; SI-NEXT: v_mov_b32_e32 v38, s72
-; SI-NEXT: v_mov_b32_e32 v49, s62
-; SI-NEXT: v_mov_b32_e32 v22, s28
-; SI-NEXT: v_mov_b32_e32 v24, s56
-; SI-NEXT: v_mov_b32_e32 v19, s46
+; SI-NEXT: v_mov_b32_e32 v27, s56
+; SI-NEXT: v_mov_b32_e32 v20, s46
; SI-NEXT: v_mov_b32_e32 v56, s44
-; SI-NEXT: v_mov_b32_e32 v42, s42
-; SI-NEXT: v_mov_b32_e32 v54, s40
-; SI-NEXT: v_mov_b32_e32 v7, s24
-; SI-NEXT: v_mov_b32_e32 v30, s22
-; SI-NEXT: v_mov_b32_e32 v21, s20
-; SI-NEXT: v_mov_b32_e32 v48, s18
-; SI-NEXT: v_mov_b32_e32 v36, s16
-; SI-NEXT: v_mov_b32_e32 v32, s14
-; SI-NEXT: v_mov_b32_e32 v44, s12
-; SI-NEXT: v_mov_b32_e32 v47, s10
-; SI-NEXT: v_mov_b32_e32 v35, s6
-; SI-NEXT: v_mov_b32_e32 v37, s4
-; SI-NEXT: v_mov_b32_e32 v52, s96
-; SI-NEXT: v_mov_b32_e32 v28, s86
-; SI-NEXT: v_mov_b32_e32 v14, s84
-; SI-NEXT: v_mov_b32_e32 v23, s82
-; SI-NEXT: v_mov_b32_e32 v27, s80
-; SI-NEXT: v_mov_b32_e32 v25, s70
-; SI-NEXT: v_mov_b32_e32 v26, s68
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_mov_b32_e32 v5, s66
-; SI-NEXT: v_mov_b32_e32 v12, s54
-; SI-NEXT: v_mov_b32_e32 v33, s50
-; SI-NEXT: v_mov_b32_e32 v8, s48
-; SI-NEXT: v_mov_b32_e32 v15, s26
+; SI-NEXT: v_mov_b32_e32 v43, s42
+; SI-NEXT: v_mov_b32_e32 v52, s40
+; SI-NEXT: v_mov_b32_e32 v37, s24
+; SI-NEXT: v_mov_b32_e32 v31, s22
+; SI-NEXT: v_mov_b32_e32 v25, s20
+; SI-NEXT: v_mov_b32_e32 v18, s18
+; SI-NEXT: v_mov_b32_e32 v13, s16
+; SI-NEXT: v_mov_b32_e32 v11, s14
+; SI-NEXT: v_mov_b32_e32 v9, s12
+; SI-NEXT: v_mov_b32_e32 v7, s10
+; SI-NEXT: v_mov_b32_e32 v5, s8
+; SI-NEXT: v_mov_b32_e32 v40, s98
+; SI-NEXT: v_mov_b32_e32 v48, s96
+; SI-NEXT: v_mov_b32_e32 v47, s86
+; SI-NEXT: v_mov_b32_e32 v33, s84
+; SI-NEXT: v_mov_b32_e32 v22, s82
; SI-NEXT: v_readlane_b32 s10, v62, 34
; SI-NEXT: v_readlane_b32 s12, v62, 35
; SI-NEXT: v_readlane_b32 s14, v62, 36
@@ -115656,19 +115225,33 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s44, v62, 48
; SI-NEXT: v_readlane_b32 s46, v62, 49
; SI-NEXT: v_readlane_b32 s56, v62, 50
+; SI-NEXT: v_readlane_b32 s58, v62, 51
; SI-NEXT: v_readlane_b32 s59, v62, 52
+; SI-NEXT: v_readlane_b32 s60, v62, 53
; SI-NEXT: v_readlane_b32 s61, v62, 54
; SI-NEXT: v_readlane_b32 s62, v62, 55
; SI-NEXT: v_readlane_b32 s63, v62, 56
; SI-NEXT: v_readlane_b32 s72, v62, 57
; SI-NEXT: v_readlane_b32 s73, v62, 58
+; SI-NEXT: v_readlane_b32 s74, v62, 59
; SI-NEXT: v_readlane_b32 s75, v62, 60
+; SI-NEXT: v_readlane_b32 s76, v62, 61
; SI-NEXT: v_readlane_b32 s77, v62, 62
+; SI-NEXT: v_readlane_b32 s78, v62, 63
; SI-NEXT: v_readlane_b32 s79, v61, 0
+; SI-NEXT: v_readlane_b32 s88, v61, 1
; SI-NEXT: v_readlane_b32 s89, v61, 2
+; SI-NEXT: v_readlane_b32 s90, v61, 3
; SI-NEXT: v_readlane_b32 s91, v61, 4
+; SI-NEXT: v_readlane_b32 s92, v61, 5
; SI-NEXT: v_readlane_b32 s93, v61, 6
+; SI-NEXT: v_readlane_b32 s94, v61, 7
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s68
; SI-NEXT: v_readlane_b32 s95, v61, 8
+; SI-NEXT: v_readlane_b32 s30, v61, 9
; SI-NEXT: v_readlane_b32 s31, v61, 10
; SI-NEXT: v_readlane_b32 s34, v61, 11
; SI-NEXT: v_readlane_b32 s35, v61, 12
@@ -115677,83 +115260,310 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s8, v61, 15
; SI-NEXT: v_readlane_b32 vcc_lo, v61, 16
; SI-NEXT: v_readlane_b32 s6, v61, 17
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s30
-; SI-NEXT: v_readlane_b32 s30, v61, 9
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s66
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s94
-; SI-NEXT: v_readlane_b32 s94, v61, 7
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s64
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s92
-; SI-NEXT: v_readlane_b32 s92, v61, 5
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s54
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s90
-; SI-NEXT: v_readlane_b32 s90, v61, 3
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s52
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s88
-; SI-NEXT: v_readlane_b32 s88, v61, 1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s50
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s78
-; SI-NEXT: v_readlane_b32 s78, v62, 63
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 2
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s76
-; SI-NEXT: v_readlane_b32 s76, v62, 61
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 4
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s74
-; SI-NEXT: v_readlane_b32 s74, v62, 59
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 6
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s60
-; SI-NEXT: v_readlane_b32 s60, v62, 53
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 8
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s58
-; SI-NEXT: v_readlane_b32 s58, v62, 51
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 10
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s98
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s52
-; SI-NEXT: v_mov_b32_e32 v1, s64
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 12
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 14
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 18
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 20
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 22
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 24
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 26
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 28
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 30
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 32
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s48
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: .LBB73_5: ; %end
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v24
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v16
; SI-NEXT: s_and_b32 s4, s57, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v15
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, vcc_lo, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v14
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v59
+; SI-NEXT: s_and_b32 s4, s47, 0xff
; SI-NEXT: s_lshl_b32 s6, s37, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v58
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s36, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v57
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s35, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 12, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v56
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v45
+; SI-NEXT: s_and_b32 s4, s45, 0xff
+; SI-NEXT: s_lshl_b32 s6, s34, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v44
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s31, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v41
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s30, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 16, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 20, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v43
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v50
+; SI-NEXT: s_and_b32 s4, s43, 0xff
+; SI-NEXT: s_lshl_b32 s6, s95, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v49
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s94, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s93, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 24, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 28, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v52
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v23
+; SI-NEXT: s_and_b32 s4, s41, 0xff
+; SI-NEXT: s_lshl_b32 s6, s92, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v38
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s91, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v54
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s90, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 32, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 36, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v37
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v34
+; SI-NEXT: s_and_b32 s4, s25, 0xff
+; SI-NEXT: s_lshl_b32 s6, s89, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v29
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s88, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v40
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s79, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 40, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 44, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v31
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v48
+; SI-NEXT: s_and_b32 s4, s23, 0xff
+; SI-NEXT: s_lshl_b32 s6, s78, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v47
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s77, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v33
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s76, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 48, v0
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 52, v0
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v25
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v22
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s21, 0xff
+; SI-NEXT: s_lshl_b32 s6, s75, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s74, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s73, 24
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s72, 8
+; SI-NEXT: s_lshl_b32 s8, s62, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_readlane_b32 s99, v63, 35
; SI-NEXT: v_readlane_b32 s98, v63, 34
; SI-NEXT: v_readlane_b32 s97, v63, 33
@@ -115785,432 +115595,264 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s39, v63, 7
; SI-NEXT: v_readlane_b32 s38, v63, 6
; SI-NEXT: v_readlane_b32 s37, v63, 5
-; SI-NEXT: v_readlane_b32 s35, v63, 3
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v10
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v19
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s4, s47, 0xff
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s36, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s34, 8
-; SI-NEXT: s_lshl_b32 s8, s30, 24
; SI-NEXT: v_readlane_b32 s36, v63, 4
+; SI-NEXT: v_readlane_b32 s35, v63, 3
; SI-NEXT: v_readlane_b32 s34, v63, 2
-; SI-NEXT: v_readlane_b32 s30, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v18
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 8, v0
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 12, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v56
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s4, s45, 0xff
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s31, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s95, 8
-; SI-NEXT: s_lshl_b32 s8, s93, 24
; SI-NEXT: v_readlane_b32 s31, v63, 1
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v55
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v55
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v55
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 16, v0
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 20, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v42
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s4, s43, 0xff
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s94, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v38
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s92, 8
-; SI-NEXT: s_lshl_b32 s8, s90, 24
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v41
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v41
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 24, v0
+; SI-NEXT: v_add_i32_e32 v4, vcc, 56, v0
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 28, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 60, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v54
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v49
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_and_b32 s4, s41, 0xff
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v18
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s19, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s91, 0xff
+; SI-NEXT: s_and_b32 s6, s63, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s89, 8
-; SI-NEXT: s_lshl_b32 s8, s79, 24
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v38
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s61, 8
+; SI-NEXT: s_lshl_b32 s8, s59, 24
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v38
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 32, v0
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 36, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v7
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v22
-; SI-NEXT: s_and_b32 s4, s25, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v13
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s88, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s78, 8
-; SI-NEXT: s_lshl_b32 s8, s76, 24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 40, v0
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 44, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v30
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v52
-; SI-NEXT: s_and_b32 s4, s23, 0xff
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v28
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s77, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v14
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 48, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 52, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v21
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v23
-; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s6, s75, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v27
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s74, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v25
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s73, 24
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 56, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 60, v0
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v48
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v26
-; SI-NEXT: s_and_b32 s4, s19, 0xff
-; SI-NEXT: s_lshl_b32 s6, s72, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v5
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s63, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s62, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_add_i32_e32 v4, vcc, 64, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x44, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v36
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v13
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s17, 0xff
-; SI-NEXT: s_lshl_b32 s6, s61, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s60, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v33
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s59, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_lshl_b32 s6, s58, 8
; SI-NEXT: s_lshl_b32 s8, s46, 24
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v20
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v37
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v6, v2
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x48, v0
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x4c, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v32
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v17
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v11
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s15, 0xff
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v34
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s56, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v45
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s44, 8
+; SI-NEXT: s_lshl_b32 s8, s40, 24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v6, v2
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x50, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x54, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v44
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v59
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v9
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s13, 0xff
-; SI-NEXT: s_lshl_b32 s6, s44, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v40
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s42, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v58
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s40, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s29, 8
+; SI-NEXT: s_lshl_b32 s8, s27, 24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x58, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x5c, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v47
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v9
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v7
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s11, 0xff
-; SI-NEXT: s_lshl_b32 s6, s29, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v53
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s28, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v50
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s27, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s26, 8
+; SI-NEXT: s_lshl_b32 s8, s22, 24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x60, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x64, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v29
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s9, 0xff
-; SI-NEXT: s_lshl_b32 s6, s26, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v31
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s24, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s22, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v5, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s20, 8
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x68, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x6c, v0
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v35
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v43
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s20, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v46
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s18, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v16
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s7, s16, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s10, 24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x70, v0
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x74, v0
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v51
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s5, 0xff
; SI-NEXT: s_lshl_b32 s5, s14, 8
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v8
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s12, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v15
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s10, 24
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x78, v0
-; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: s_waitcnt expcnt(0)
@@ -117054,14 +116696,13 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_add_u32_e32 v9, vcc, 52, v0
; VI-NEXT: v_mov_b32_e32 v15, s4
; VI-NEXT: buffer_store_dword v15, v9, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v36
-; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: s_and_b32 s4, s23, 0xff
; VI-NEXT: s_lshl_b32 s6, s75, 8
; VI-NEXT: s_or_b32 s4, s4, s6
; VI-NEXT: s_and_b32 s6, s74, 0xff
; VI-NEXT: s_lshl_b32 s8, s73, 8
+; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v36
; VI-NEXT: s_or_b32 s6, s6, s8
; VI-NEXT: v_or_b32_sdwa v9, v19, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -117101,8 +116742,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readlane_b32 s34, v63, 2
; VI-NEXT: v_readlane_b32 s31, v63, 1
; VI-NEXT: v_readlane_b32 s30, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v15, 8, v36
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; VI-NEXT: v_or_b32_sdwa v15, v34, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v15, vcc, 56, v0
@@ -117204,21 +116845,20 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v9, s4
; VI-NEXT: buffer_store_dword v9, v7, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v12
-; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; VI-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: s_and_b32 s4, s9, 0xff
; VI-NEXT: s_lshl_b32 s6, s26, 8
; VI-NEXT: s_or_b32 s4, s4, s6
; VI-NEXT: s_and_b32 s6, s24, 0xff
; VI-NEXT: s_lshl_b32 s8, s22, 8
-; VI-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_or_b32 s6, s6, s8
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s6, s6, 16
; VI-NEXT: s_or_b32 s4, s4, s6
; VI-NEXT: s_lshl_b32 s6, s16, 8
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v11
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; VI-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x68, v0
@@ -117227,21 +116867,20 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v7, s4
; VI-NEXT: buffer_store_dword v7, v5, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v8
-; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: s_and_b32 s4, s5, 0xff
; VI-NEXT: s_lshl_b32 s5, s20, 8
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, s18, 0xff
-; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_or_b32 s5, s5, s6
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_lshl_b32 s5, s14, 8
; VI-NEXT: s_lshl_b32 s6, s10, 8
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v7
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; VI-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x70, v0
@@ -117252,7 +116891,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v4
; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; VI-NEXT: s_and_b32 s4, s7, 0xff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, s12, 0xff
@@ -117260,7 +116898,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s4, s5
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; VI-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -118151,14 +117789,13 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v11, s4
; GFX9-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:84
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v20
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b32 s4, s13, 0xff
; GFX9-NEXT: s_lshl_b32 s6, s44, 8
; GFX9-NEXT: s_or_b32 s4, s4, s6
; GFX9-NEXT: s_and_b32 s6, s42, 0xff
; GFX9-NEXT: s_lshl_b32 s8, s40, 8
-; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_or_b32 s6, s6, s8
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s6, s6, 16
@@ -118201,8 +117838,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readlane_b32 s34, v63, 2
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v19
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: v_or_b32_sdwa v11, v18, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:88
@@ -118224,42 +117861,40 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v7, s4
; GFX9-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:100
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v12
-; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b32 s4, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s6, s26, 8
; GFX9-NEXT: s_or_b32 s4, s4, s6
; GFX9-NEXT: s_and_b32 s6, s24, 0xff
; GFX9-NEXT: s_lshl_b32 s8, s22, 8
-; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_or_b32 s6, s6, s8
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s6, s6, 16
; GFX9-NEXT: s_or_b32 s4, s4, s6
; GFX9-NEXT: s_lshl_b32 s6, s16, 8
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v11
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:104
; GFX9-NEXT: v_mov_b32_e32 v5, s4
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:108
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v8
-; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b32 s4, s5, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s20, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s18, 0xff
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_or_b32 s5, s5, s6
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s5, s5, 16
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_lshl_b32 s5, s14, 8
; GFX9-NEXT: s_lshl_b32 s6, s10, 8
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:112
@@ -118268,7 +117903,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v4
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b32 s4, s7, 0xff
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s12, 0xff
@@ -118276,7 +117910,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
; GFX9-NEXT: s_lshl_b32 s5, s5, 16
; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -135259,19 +134893,18 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshr_b64 v[29:30], v[53:54], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v52, v53
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshr_b64 v[30:31], v[31:32], 16
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v57
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v53, v31
; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
; SI-NEXT: s_branch .LBB79_3
@@ -135337,76 +134970,62 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v42, v51
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB79_3: ; %Flow
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v56
; SI-NEXT: v_mov_b32_e32 v37, v34
; SI-NEXT: v_mov_b32_e32 v34, v33
-; SI-NEXT: v_mov_b32_e32 v35, v56
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v32, v40
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mov_b32_e32 v33, v38
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v51, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v54, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v44, v46
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v45, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v47, v56
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v58, v60
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mov_b32_e32 v32, v40
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_cbranch_vccnz .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v61
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -135416,6 +135035,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
@@ -135425,15 +135045,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -135445,15 +135058,19 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
@@ -135461,37 +135078,55 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -135537,19 +135172,13 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -135561,78 +135190,42 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
@@ -135644,15 +135237,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
; SI-NEXT: .LBB79_5: ; %end
@@ -159876,11 +159463,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_or_b32_sdwa v1, v1, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v44, 8, v43
-; VI-NEXT: v_or_b32_sdwa v2, v2, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; VI-NEXT: v_or_b32_sdwa v2, v2, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v43, 8, v43
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v43, v44, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -159896,9 +159482,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 4, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -159923,9 +159508,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 12, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -159944,10 +159528,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 20, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v60
; VI-NEXT: v_or_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v59, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -159961,10 +159544,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 28, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v41
; VI-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v51, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -159978,10 +159560,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 36, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v46
; VI-NEXT: v_or_b32_sdwa v2, v11, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v39, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -159995,9 +159576,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160022,9 +159602,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 52, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160049,9 +159628,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 60, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160076,9 +159654,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x44, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160104,11 +159681,11 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160130,11 +159707,11 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -160180,11 +159757,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -160206,10 +159782,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x6c, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v34
; VI-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -160223,10 +159798,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x74, v0
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v63
; VI-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v58, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161508,16 +161082,6 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b16_e32 v32, 8, v32
-; GFX9-NEXT: v_lshlrev_b16_e32 v31, 8, v31
-; GFX9-NEXT: v_or_b32_sdwa v5, v5, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v32, 8, v42
-; GFX9-NEXT: v_or_b32_sdwa v7, v7, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v31, 8, v39
-; GFX9-NEXT: v_or_b32_sdwa v6, v6, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b16_e32 v41, 8, v41
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b16_e32 v51, 8, v51
@@ -161526,7 +161090,15 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b16_e32 v48, 8, v52
; GFX9-NEXT: v_or_b32_sdwa v4, v4, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshlrev_b16_e32 v32, 8, v32
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b16_e32 v32, 8, v42
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b16_e32 v31, 8, v31
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b16_e32 v31, 8, v39
+; GFX9-NEXT: v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 8, v11
; GFX9-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
@@ -161549,7 +161121,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b16_e32 v17, 8, v60
; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v18, 8, v31
+; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b16_e32 v18, 8, v18
; GFX9-NEXT: v_or_b32_sdwa v18, v38, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v1, v46, s[0:3], 0 offen
@@ -161561,9 +161135,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v46, s[0:3], 0 offen offset:4
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161580,9 +161153,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v46, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161599,9 +161171,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v46, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161618,9 +161189,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v46, s[0:3], 0 offen offset:28
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161637,9 +161207,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161656,9 +161225,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161675,9 +161243,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -161693,9 +161260,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v0, v15, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v61, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161707,11 +161272,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v0, v17, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v45
; GFX9-NEXT: v_or_b32_sdwa v1, v19, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v34, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161725,10 +161288,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:76
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v55
; GFX9-NEXT: v_or_b32_sdwa v0, v21, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v56, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161742,10 +161304,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v0, v46, s[0:3], 0 offen offset:84
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v57
; GFX9-NEXT: v_or_b32_sdwa v0, v23, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v36, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161763,12 +161324,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161789,12 +161350,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161815,12 +161376,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -161841,12 +161402,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -164252,30 +163813,31 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:80
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:12
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:12
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:20
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:16
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16
; SI-NEXT: s_waitcnt expcnt(3)
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:28
; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:24
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:44
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:64
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:72
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:72
; SI-NEXT: v_writelane_b32 v63, s30, 0
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_writelane_b32 v63, s34, 2
@@ -164308,323 +163870,248 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v63, s85, 29
; SI-NEXT: v_writelane_b32 v63, s86, 30
; SI-NEXT: v_writelane_b32 v63, s87, 31
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v10
+; SI-NEXT: v_mul_f32_e32 v37, 1.0, v10
; SI-NEXT: v_writelane_b32 v63, s96, 32
; SI-NEXT: v_writelane_b32 v63, s97, 33
; SI-NEXT: v_writelane_b32 v63, s98, 34
; SI-NEXT: v_writelane_b32 v63, s99, 35
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v3
+; SI-NEXT: v_mul_f32_e32 v53, 1.0, v2
+; SI-NEXT: v_mul_f32_e32 v45, 1.0, v1
+; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v7
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v17
+; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; SI-NEXT: v_mul_f32_e32 v59, 1.0, v8
+; SI-NEXT: v_mul_f32_e32 v46, 1.0, v7
+; SI-NEXT: v_mul_f32_e32 v8, 1.0, v9
+; SI-NEXT: v_mul_f32_e32 v7, 1.0, v12
+; SI-NEXT: v_mul_f32_e32 v9, 1.0, v11
+; SI-NEXT: v_mul_f32_e32 v49, 1.0, v14
+; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
+; SI-NEXT: v_mul_f32_e32 v11, 1.0, v16
+; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
+; SI-NEXT: v_mul_f32_e32 v33, 1.0, v18
; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v23
+; SI-NEXT: v_mul_f32_e32 v15, 1.0, v19
+; SI-NEXT: v_mul_f32_e32 v12, 1.0, v22
+; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: v_mul_f32_e32 v31, 1.0, v17
+; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; SI-NEXT: v_mul_f32_e32 v18, 1.0, v25
; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v38
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v48
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v34
-; SI-NEXT: s_waitcnt vmcnt(8) expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v54
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e32 v25, 1.0, v27
+; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; SI-NEXT: v_mul_f32_e32 v22, 1.0, v29
+; SI-NEXT: v_mul_f32_e32 v27, 1.0, v32
+; SI-NEXT: v_mul_f32_e32 v16, 1.0, v34
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: s_waitcnt vmcnt(6) expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v42
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v57
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v59
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v41
+; SI-NEXT: v_mul_f32_e32 v29, 1.0, v35
+; SI-NEXT: v_mul_f32_e32 v34, 1.0, v36
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_mul_f32_e32 v10, 1.0, v51
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(7) expcnt(0)
+; SI-NEXT: v_mul_f32_e32 v10, 1.0, v52
; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(6) expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v43
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: v_mul_f32_e32 v10, 1.0, v40
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: v_mul_f32_e32 v10, 1.0, v41
+; SI-NEXT: v_mul_f32_e32 v35, 1.0, v38
+; SI-NEXT: v_mul_f32_e32 v36, 1.0, v56
+; SI-NEXT: v_mul_f32_e32 v56, 1.0, v57
+; SI-NEXT: v_mul_f32_e32 v57, 1.0, v58
+; SI-NEXT: v_mul_f32_e32 v58, 1.0, v60
+; SI-NEXT: v_mul_f32_e32 v39, 1.0, v39
+; SI-NEXT: v_mul_f32_e32 v32, 1.0, v48
+; SI-NEXT: v_mul_f32_e32 v50, 1.0, v50
+; SI-NEXT: v_mul_f32_e32 v54, 1.0, v54
+; SI-NEXT: v_mul_f32_e32 v48, 1.0, v55
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v44
+; SI-NEXT: v_mul_f32_e32 v44, 1.0, v42
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v45
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s25
+; SI-NEXT: v_mul_f32_e32 v47, 1.0, v43
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v42, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s26
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s26
; SI-NEXT: v_mul_f32_e64 v43, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s28
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s28
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr61 : SGPR spill to VGPR lane
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readfirstlane_b32 s4, v13
+; SI-NEXT: v_readfirstlane_b32 s4, v38
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v52
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s4, v41
; SI-NEXT: s_lshr_b32 s7, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v30
+; SI-NEXT: v_readfirstlane_b32 s4, v13
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v34
+; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v54
+; SI-NEXT: v_readfirstlane_b32 s4, v51
; SI-NEXT: s_lshr_b32 s65, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v41
+; SI-NEXT: v_readfirstlane_b32 s4, v55
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v42
; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v45
+; SI-NEXT: v_readfirstlane_b32 s4, v40
; SI-NEXT: s_lshr_b32 s69, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v43
+; SI-NEXT: v_readfirstlane_b32 s56, v5
+; SI-NEXT: v_mov_b32_e32 v5, v15
+; SI-NEXT: v_mov_b32_e32 v15, v24
+; SI-NEXT: v_mov_b32_e32 v24, v18
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v44
-; SI-NEXT: v_mov_b32_e32 v34, v35
+; SI-NEXT: v_readfirstlane_b32 s4, v60
+; SI-NEXT: v_mov_b32_e32 v38, v53
+; SI-NEXT: v_readfirstlane_b32 s88, v8
+; SI-NEXT: v_mov_b32_e32 v8, v22
+; SI-NEXT: v_mov_b32_e32 v22, v33
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b64 s[66:67], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v34
+; SI-NEXT: v_readfirstlane_b32 s4, v38
+; SI-NEXT: v_mov_b32_e32 v52, v4
; SI-NEXT: s_lshr_b32 s91, s4, 16
-; SI-NEXT: v_mov_b32_e32 v30, v51
-; SI-NEXT: v_readfirstlane_b32 s4, v47
-; SI-NEXT: v_mov_b32_e32 v51, v46
+; SI-NEXT: v_readfirstlane_b32 s4, v52
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v51
-; SI-NEXT: v_mov_b32_e32 v35, v6
+; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: v_mov_b32_e32 v53, v6
; SI-NEXT: s_lshr_b64 s[52:53], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v35
+; SI-NEXT: v_readfirstlane_b32 s4, v53
; SI-NEXT: s_lshr_b32 s37, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v1
+; SI-NEXT: v_readfirstlane_b32 s4, v59
+; SI-NEXT: v_mov_b32_e32 v59, v46
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_mov_b32_e32 v60, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_mov_b32_e32 v41, v5
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: v_mov_b32_e32 v5, v39
+; SI-NEXT: v_readfirstlane_b32 s4, v59
+; SI-NEXT: v_mov_b32_e32 v6, v37
; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v5
+; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_lshr_b32 s89, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v9
+; SI-NEXT: v_readfirstlane_b32 s4, v7
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v31
+; SI-NEXT: v_readfirstlane_b32 s4, v9
; SI-NEXT: s_lshr_b64 s[50:51], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: v_readfirstlane_b32 s4, v49
; SI-NEXT: s_lshr_b32 s57, s4, 16
-; SI-NEXT: v_mov_b32_e32 v42, v32
-; SI-NEXT: v_readfirstlane_b32 s4, v32
-; SI-NEXT: v_mov_b32_e32 v32, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v11
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v32
-; SI-NEXT: v_mov_b32_e32 v6, v55
+; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_lshr_b64 s[92:93], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v6
+; SI-NEXT: v_readfirstlane_b32 s4, v22
; SI-NEXT: s_lshr_b32 s79, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: v_mov_b32_e32 v39, v12
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v39
-; SI-NEXT: v_mov_b32_e32 v9, v8
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mov_b32_e32 v43, v20
+; SI-NEXT: v_readfirstlane_b32 s4, v5
; SI-NEXT: s_lshr_b64 s[76:77], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: v_mov_b32_e32 v20, v21
-; SI-NEXT: v_readfirstlane_b32 s78, v18
+; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: v_mov_b32_e32 v18, v22
+; SI-NEXT: v_readfirstlane_b32 s4, v15
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v23
+; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s4, v26
; SI-NEXT: s_lshr_b32 s59, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: v_mov_b32_e32 v21, v25
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v27
-; SI-NEXT: v_mov_b32_e32 v25, v26
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: v_mov_b32_e32 v12, v29
+; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s4, v30
; SI-NEXT: s_lshr_b32 s45, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v12
-; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_mov_b32_e32 v44, v1
+; SI-NEXT: v_readfirstlane_b32 s4, v27
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_mov_b32_e32 v1, v52
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v52, v17
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v29, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v22, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v29
; SI-NEXT: s_lshr_b32 s29, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: v_readfirstlane_b32 s12, v40
-; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
-; SI-NEXT: s_mov_b32 s9, s96
-; SI-NEXT: v_readfirstlane_b32 s88, v60
-; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
-; SI-NEXT: v_readfirstlane_b32 s64, v16
-; SI-NEXT: s_lshr_b64 s[84:85], s[64:65], 16
-; SI-NEXT: s_mov_b32 s87, s84
-; SI-NEXT: v_readfirstlane_b32 s68, v48
-; SI-NEXT: s_lshr_b64 s[70:71], s[68:69], 16
-; SI-NEXT: s_mov_b32 s81, s70
-; SI-NEXT: v_readfirstlane_b32 s90, v30
-; SI-NEXT: s_lshr_b64 s[38:39], s[90:91], 16
-; SI-NEXT: s_mov_b32 s67, s38
-; SI-NEXT: v_readfirstlane_b32 s36, v3
-; SI-NEXT: s_lshr_b64 s[98:99], s[36:37], 16
-; SI-NEXT: s_mov_b32 s53, s98
-; SI-NEXT: s_mov_b32 s31, s82
-; SI-NEXT: v_readfirstlane_b32 s56, v7
-; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
-; SI-NEXT: s_mov_b32 s51, s94
-; SI-NEXT: s_lshr_b64 s[74:75], s[78:79], 16
-; SI-NEXT: s_mov_b32 s93, s74
-; SI-NEXT: v_readfirstlane_b32 s72, v19
-; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
-; SI-NEXT: s_mov_b32 s77, s60
-; SI-NEXT: v_readfirstlane_b32 s58, v21
-; SI-NEXT: s_lshr_b64 s[54:55], s[58:59], 16
-; SI-NEXT: s_mov_b32 s63, s54
-; SI-NEXT: v_readfirstlane_b32 s44, v22
-; SI-NEXT: s_lshr_b64 s[42:43], s[44:45], 16
-; SI-NEXT: s_mov_b32 s47, s42
-; SI-NEXT: v_mov_b32_e32 v26, v37
-; SI-NEXT: v_readfirstlane_b32 s28, v26
-; SI-NEXT: s_lshr_b64 s[26:27], s[28:29], 16
-; SI-NEXT: s_mov_b32 s41, s26
-; SI-NEXT: v_readfirstlane_b32 s22, v36
-; SI-NEXT: v_readfirstlane_b32 s18, v49
-; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v1
-; SI-NEXT: v_mov_b32_e32 v1, v56
-; SI-NEXT: v_mov_b32_e32 v3, v54
-; SI-NEXT: v_lshrrev_b32_e32 v37, 24, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v50
-; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v38
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v1
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v54, v59
-; SI-NEXT: s_lshr_b32 s78, s96, 8
-; SI-NEXT: s_lshr_b32 s61, s84, 8
-; SI-NEXT: s_lshr_b32 s72, s70, 8
-; SI-NEXT: s_lshr_b32 s75, s38, 8
-; SI-NEXT: s_lshr_b32 s58, s98, 8
-; SI-NEXT: s_lshr_b32 s43, s82, 8
-; SI-NEXT: s_lshr_b32 s44, s94, 8
-; SI-NEXT: s_mov_b32 s64, s74
-; SI-NEXT: s_lshr_b32 s27, s74, 8
-; SI-NEXT: s_mov_b32 s90, s60
-; SI-NEXT: s_lshr_b32 s28, s60, 8
-; SI-NEXT: s_lshr_b32 s74, s54, 8
-; SI-NEXT: s_mov_b32 s68, s42
-; SI-NEXT: s_mov_b32 s56, s26
-; SI-NEXT: v_lshrrev_b32_e32 v16, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v45
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v34
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v35
-; SI-NEXT: v_lshrrev_b32_e32 v47, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v25
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_readfirstlane_b32 s4, v35
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s4, v33
+; SI-NEXT: v_readfirstlane_b32 s4, v36
; SI-NEXT: s_lshr_b64 s[24:25], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v57
+; SI-NEXT: v_readfirstlane_b32 s4, v56
; SI-NEXT: s_lshr_b32 s23, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v58
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v59
+; SI-NEXT: v_readfirstlane_b32 s4, v39
; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v50
+; SI-NEXT: v_readfirstlane_b32 s4, v32
; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v53
-; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v17
+; SI-NEXT: v_readfirstlane_b32 s12, v48
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s4, v24
+; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
+; SI-NEXT: s_mov_b32 s9, s96
+; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
+; SI-NEXT: v_readfirstlane_b32 s64, v10
+; SI-NEXT: s_lshr_b64 s[84:85], s[64:65], 16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_readfirstlane_b32 s4, v33
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v38
+; SI-NEXT: v_readfirstlane_b32 s4, v54
; SI-NEXT: s_lshr_b32 s13, s4, 16
; SI-NEXT: s_mov_b32 s5, s13
; SI-NEXT: v_writelane_b32 v61, s4, 26
; SI-NEXT: v_writelane_b32 v61, s5, 27
+; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
+; SI-NEXT: v_readfirstlane_b32 s12, v47
+; SI-NEXT: s_mov_b32 s87, s84
+; SI-NEXT: v_readfirstlane_b32 s68, v19
+; SI-NEXT: s_lshr_b64 s[70:71], s[68:69], 16
+; SI-NEXT: s_mov_b32 s81, s70
+; SI-NEXT: v_readfirstlane_b32 s90, v45
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_readfirstlane_b32 s4, v46
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s4, v21
; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s5, v56
-; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
+; SI-NEXT: v_readfirstlane_b32 s5, v44
; SI-NEXT: s_lshr_b32 s13, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: s_lshr_b64 vcc, s[12:13], 16
; SI-NEXT: s_mov_b32 s5, vcc_lo
; SI-NEXT: s_mov_b32 s88, vcc_lo
@@ -164652,17 +164139,22 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 17
; SI-NEXT: s_lshr_b64 vcc, s[80:81], 16
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 14
+; SI-NEXT: s_lshr_b64 s[38:39], s[90:91], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 15
; SI-NEXT: s_lshr_b64 vcc, s[80:81], 8
+; SI-NEXT: s_mov_b32 s67, s38
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 12
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 13
; SI-NEXT: s_lshr_b64 vcc, s[66:67], 24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 22
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 23
; SI-NEXT: s_lshr_b64 vcc, s[66:67], 16
+; SI-NEXT: v_readfirstlane_b32 s36, v1
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 20
+; SI-NEXT: s_lshr_b64 s[98:99], s[36:37], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 21
; SI-NEXT: s_lshr_b64 vcc, s[66:67], 8
+; SI-NEXT: s_mov_b32 s53, s98
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 18
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 19
; SI-NEXT: s_lshr_b64 vcc, s[52:53], 24
@@ -164672,6 +164164,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 26
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 27
; SI-NEXT: s_lshr_b64 vcc, s[52:53], 8
+; SI-NEXT: s_mov_b32 s31, s82
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 24
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 25
; SI-NEXT: s_lshr_b64 vcc, s[30:31], 24
@@ -164679,59 +164172,77 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 35
; SI-NEXT: s_lshr_b64 vcc, s[30:31], 16
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 32
+; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 33
; SI-NEXT: s_lshr_b64 vcc, s[30:31], 8
+; SI-NEXT: s_mov_b32 s51, s94
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 30
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 31
; SI-NEXT: s_lshr_b64 vcc, s[50:51], 24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 40
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 41
; SI-NEXT: s_lshr_b64 vcc, s[50:51], 16
+; SI-NEXT: v_readfirstlane_b32 s78, v31
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 38
+; SI-NEXT: s_lshr_b64 s[74:75], s[78:79], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 39
; SI-NEXT: s_lshr_b64 vcc, s[50:51], 8
+; SI-NEXT: s_mov_b32 s93, s74
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 36
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 37
; SI-NEXT: s_lshr_b64 vcc, s[92:93], 24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 46
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 47
; SI-NEXT: s_lshr_b64 vcc, s[92:93], 16
+; SI-NEXT: v_readfirstlane_b32 s72, v60
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 44
+; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 45
; SI-NEXT: s_lshr_b64 vcc, s[92:93], 8
+; SI-NEXT: s_mov_b32 s77, s60
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 42
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 43
; SI-NEXT: s_lshr_b64 vcc, s[76:77], 24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 52
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 53
; SI-NEXT: s_lshr_b64 vcc, s[76:77], 16
+; SI-NEXT: v_readfirstlane_b32 s58, v24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 50
+; SI-NEXT: s_lshr_b64 s[54:55], s[58:59], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 51
; SI-NEXT: s_lshr_b64 vcc, s[76:77], 8
+; SI-NEXT: s_mov_b32 s63, s54
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 48
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 49
; SI-NEXT: s_lshr_b64 vcc, s[62:63], 24
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 58
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 59
; SI-NEXT: s_lshr_b64 vcc, s[62:63], 16
+; SI-NEXT: v_readfirstlane_b32 s44, v8
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 56
+; SI-NEXT: s_lshr_b64 s[42:43], s[44:45], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 57
; SI-NEXT: s_lshr_b64 vcc, s[62:63], 8
+; SI-NEXT: s_mov_b32 s47, s42
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 54
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 55
; SI-NEXT: s_lshr_b64 vcc, s[46:47], 24
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 0
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 1
; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
+; SI-NEXT: v_readfirstlane_b32 s28, v34
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 62
+; SI-NEXT: s_lshr_b64 s[26:27], s[28:29], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 63
; SI-NEXT: s_lshr_b64 vcc, s[46:47], 8
+; SI-NEXT: s_mov_b32 s41, s26
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 60
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 61
; SI-NEXT: s_lshr_b64 vcc, s[40:41], 24
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 6
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 7
; SI-NEXT: s_lshr_b64 vcc, s[40:41], 16
+; SI-NEXT: v_readfirstlane_b32 s22, v57
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 4
; SI-NEXT: s_lshr_b64 s[34:35], s[22:23], 16
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 5
@@ -164743,6 +164254,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 12
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 13
; SI-NEXT: s_lshr_b64 vcc, s[24:25], 16
+; SI-NEXT: v_readfirstlane_b32 s18, v50
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 10
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 11
@@ -164774,12 +164286,40 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 33
; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 30
+; SI-NEXT: v_mov_b32_e32 v43, v2
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 31
; SI-NEXT: s_lshr_b64 vcc, s[4:5], 8
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v40
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v12
+; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v56
+; SI-NEXT: v_mov_b32_e32 v42, v11
; SI-NEXT: v_writelane_b32 v61, vcc_lo, 28
-; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v23
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v38
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v29
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v32
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v54
+; SI-NEXT: v_mov_b32_e32 v4, v3
+; SI-NEXT: s_lshr_b32 s78, s96, 8
+; SI-NEXT: s_lshr_b32 s61, s84, 8
+; SI-NEXT: s_lshr_b32 s72, s70, 8
+; SI-NEXT: s_lshr_b32 s75, s38, 8
+; SI-NEXT: s_lshr_b32 s58, s98, 8
+; SI-NEXT: s_lshr_b32 s43, s82, 8
+; SI-NEXT: s_lshr_b32 s44, s94, 8
+; SI-NEXT: s_mov_b32 s64, s74
+; SI-NEXT: s_lshr_b32 s27, s74, 8
+; SI-NEXT: s_mov_b32 s90, s60
+; SI-NEXT: s_lshr_b32 s28, s60, 8
+; SI-NEXT: s_lshr_b32 s74, s54, 8
+; SI-NEXT: s_mov_b32 s68, s42
; SI-NEXT: s_lshr_b32 s22, s42, 8
+; SI-NEXT: s_mov_b32 s56, s26
; SI-NEXT: s_lshr_b32 s21, s26, 8
; SI-NEXT: s_lshr_b32 s18, s34, 8
; SI-NEXT: s_mov_b32 s36, s14
@@ -164788,182 +164328,156 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b32 s12, s20, 8
; SI-NEXT: v_writelane_b32 v61, vcc_hi, 29
; SI-NEXT: s_mov_b64 vcc, 0
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v29
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v57
-; SI-NEXT: v_mov_b32_e32 v59, v30
-; SI-NEXT: v_mov_b32_e32 v31, v51
-; SI-NEXT: v_mov_b32_e32 v60, v34
-; SI-NEXT: v_mov_b32_e32 v30, v39
-; SI-NEXT: v_mov_b32_e32 v19, v5
-; SI-NEXT: v_mov_b32_e32 v39, v21
-; SI-NEXT: v_mov_b32_e32 v21, v20
-; SI-NEXT: v_mov_b32_e32 v34, v18
-; SI-NEXT: v_mov_b32_e32 v18, v37
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v7, v26
-; SI-NEXT: v_mov_b32_e32 v20, v2
-; SI-NEXT: v_mov_b32_e32 v37, v17
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: v_mov_b32_e32 v17, v9
-; SI-NEXT: v_mov_b32_e32 v9, v10
-; SI-NEXT: v_mov_b32_e32 v26, v25
+; SI-NEXT: v_lshrrev_b32_e32 v19, 24, v41
+; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v51
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v41, 24, v53
+; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v6
+; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v49
+; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v22
+; SI-NEXT: v_lshrrev_b32_e32 v9, 24, v26
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v30
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v47, v44
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v44
+; SI-NEXT: v_mov_b32_e32 v44, v52
+; SI-NEXT: v_mov_b32_e32 v52, v4
+; SI-NEXT: v_mov_b32_e32 v4, v5
+; SI-NEXT: v_mov_b32_e32 v5, v11
+; SI-NEXT: v_mov_b32_e32 v11, v24
+; SI-NEXT: v_mov_b32_e32 v24, v15
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v55, v8
+; SI-NEXT: v_mov_b32_e32 v15, v46
+; SI-NEXT: v_mov_b32_e32 v46, v21
; SI-NEXT: s_branch .LBB91_3
; SI-NEXT: .LBB91_2:
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v54, v59
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; kill: killed $vgpr55
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v47, v44
; SI-NEXT: v_writelane_b32 v62, s4, 0
; SI-NEXT: v_writelane_b32 v62, s5, 1
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v59, v51
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; kill: killed $vgpr55
+; SI-NEXT: v_mov_b32_e32 v44, v4
; SI-NEXT: v_writelane_b32 v62, s4, 2
; SI-NEXT: v_writelane_b32 v62, s5, 3
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v31, v46
+; SI-NEXT: v_mov_b32_e32 v59, v46
; SI-NEXT: v_writelane_b32 v62, s4, 4
; SI-NEXT: v_writelane_b32 v62, s5, 5
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v34, v22
+; SI-NEXT: v_mov_b32_e32 v4, v15
; SI-NEXT: v_writelane_b32 v62, s4, 6
; SI-NEXT: v_writelane_b32 v62, s5, 7
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v22, v24
+; SI-NEXT: v_mov_b32_e32 v55, v22
; SI-NEXT: v_writelane_b32 v62, s4, 8
; SI-NEXT: v_writelane_b32 v62, s5, 9
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v7, v37
+; SI-NEXT: v_mov_b32_e32 v22, v33
; SI-NEXT: v_writelane_b32 v62, s4, 10
; SI-NEXT: v_writelane_b32 v62, s5, 11
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 12
; SI-NEXT: v_writelane_b32 v62, s5, 13
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v42, v11
; SI-NEXT: v_writelane_b32 v62, s4, 14
; SI-NEXT: v_writelane_b32 v62, s5, 15
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: v_mov_b32_e32 v44, v1
+; SI-NEXT: v_mov_b32_e32 v11, v18
; SI-NEXT: v_writelane_b32 v62, s4, 16
; SI-NEXT: v_writelane_b32 v62, s5, 17
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mov_b32_e32 v52, v17
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 18
; SI-NEXT: v_writelane_b32 v62, s5, 19
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v43, v20
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: v_mov_b32_e32 v43, v2
; SI-NEXT: v_writelane_b32 v62, s4, 20
; SI-NEXT: v_writelane_b32 v62, s5, 21
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v42, v32
+; SI-NEXT: s_mov_b64 vcc, -1
; SI-NEXT: v_writelane_b32 v62, s4, 22
; SI-NEXT: v_writelane_b32 v62, s5, 23
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v41, v5
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: v_mov_b32_e32 v52, v3
; SI-NEXT: v_writelane_b32 v62, s4, 24
; SI-NEXT: v_writelane_b32 v62, s5, 25
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_mov_b64 vcc, -1
+; SI-NEXT: v_mov_b32_e32 v38, v53
; SI-NEXT: v_writelane_b32 v62, s4, 26
; SI-NEXT: v_writelane_b32 v62, s5, 27
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v60, v35
+; SI-NEXT: v_mov_b32_e32 v53, v6
; SI-NEXT: v_writelane_b32 v62, s4, 28
; SI-NEXT: v_writelane_b32 v62, s5, 29
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: v_mov_b32_e32 v35, v6
+; SI-NEXT: v_mov_b32_e32 v60, v21
; SI-NEXT: v_writelane_b32 v62, s4, 30
; SI-NEXT: v_writelane_b32 v62, s5, 31
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v32, v4
+; SI-NEXT: v_mov_b32_e32 v6, v37
; SI-NEXT: v_writelane_b32 v62, s4, 32
; SI-NEXT: v_writelane_b32 v62, s5, 33
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v30, v12
-; SI-NEXT: v_writelane_b32 v62, s4, 34
-; SI-NEXT: v_writelane_b32 v62, s5, 35
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v19, v39
-; SI-NEXT: v_writelane_b32 v62, s4, 36
-; SI-NEXT: v_writelane_b32 v62, s5, 37
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mov_b32_e32 v39, v25
-; SI-NEXT: v_writelane_b32 v62, s4, 38
-; SI-NEXT: v_writelane_b32 v62, s5, 39
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v12, v29
-; SI-NEXT: v_writelane_b32 v62, s4, 40
-; SI-NEXT: v_writelane_b32 v62, s5, 41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v20, v2
-; SI-NEXT: v_writelane_b32 v62, s4, 42
-; SI-NEXT: v_writelane_b32 v62, s5, 43
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v6, v55
-; SI-NEXT: v_writelane_b32 v62, s4, 44
-; SI-NEXT: v_writelane_b32 v62, s5, 45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v17, v8
-; SI-NEXT: v_writelane_b32 v62, s4, 46
-; SI-NEXT: v_writelane_b32 v62, s5, 47
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v29, v33
-; SI-NEXT: v_writelane_b32 v62, s4, 48
-; SI-NEXT: v_writelane_b32 v62, s5, 49
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr96
; SI-NEXT: ; implicit-def: $sgpr78
; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $sgpr86
; SI-NEXT: ; implicit-def: $sgpr84
; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr38
; SI-NEXT: ; implicit-def: $sgpr75
; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $sgpr52
; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr82
; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $sgpr28
; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $sgpr15
; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; kill: killed $vgpr2
; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr57
@@ -164991,11 +164505,35 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_writelane_b32 v62, s4, 34
+; SI-NEXT: v_writelane_b32 v62, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: v_writelane_b32 v62, s4, 36
+; SI-NEXT: v_writelane_b32 v62, s5, 37
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 38
+; SI-NEXT: v_writelane_b32 v62, s5, 39
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 40
+; SI-NEXT: v_writelane_b32 v62, s5, 41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 42
+; SI-NEXT: v_writelane_b32 v62, s5, 43
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 44
+; SI-NEXT: v_writelane_b32 v62, s5, 45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 46
+; SI-NEXT: v_writelane_b32 v62, s5, 47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 48
+; SI-NEXT: v_writelane_b32 v62, s5, 49
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: v_writelane_b32 v62, s4, 50
; SI-NEXT: v_writelane_b32 v62, s5, 51
; SI-NEXT: ; implicit-def: $sgpr4
@@ -165068,151 +164606,167 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s89, 33
; SI-NEXT: ; implicit-def: $sgpr88
; SI-NEXT: .LBB91_3: ; %Flow
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, vcc
+; SI-NEXT: v_mov_b32_e32 v21, v9
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v45, v31
+; SI-NEXT: v_mov_b32_e32 v31, v13
; SI-NEXT: s_cbranch_vccnz .LBB91_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s4, v11
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v15
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v46
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: v_readfirstlane_b32 s4, v8
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v10
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s12, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v54
+; SI-NEXT: v_readfirstlane_b32 s6, v10
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v38
+; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v9
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v40
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
+; SI-NEXT: v_readfirstlane_b32 s18, v5
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v35
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s5, v15
+; SI-NEXT: s_lshr_b32 s13, s5, 16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s6, v8
+; SI-NEXT: v_readfirstlane_b32 s12, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v33
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v48
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v54
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_mov_b32 s7, s9
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v58
+; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_writelane_b32 v61, s6, 26
-; SI-NEXT: s_lshr_b64 s[20:21], s[8:9], 16
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v39
; SI-NEXT: v_writelane_b32 v61, s7, 27
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s6, v7
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
+; SI-NEXT: s_lshr_b64 s[20:21], s[8:9], 16
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_readfirstlane_b32 s18, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
-; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v2
+; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_lshr_b32 s19, s6, 16
+; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_lshr_b64 s[26:27], s[18:19], 16
-; SI-NEXT: s_mov_b32 s17, s26
-; SI-NEXT: s_mov_b32 s11, s20
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_readfirstlane_b32 s8, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_readfirstlane_b32 s22, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v52
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
+; SI-NEXT: s_lshr_b32 s19, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v56
+; SI-NEXT: v_readfirstlane_b32 s22, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
+; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v7
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v27
+; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v1
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s8, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v34
; SI-NEXT: s_lshr_b32 s23, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: v_readfirstlane_b32 s28, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v27
-; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v7
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v5
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
+; SI-NEXT: v_readfirstlane_b32 s28, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v25
+; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v28
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v22
-; SI-NEXT: v_readfirstlane_b32 s6, v5
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v28
+; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
+; SI-NEXT: v_readfirstlane_b32 s6, v5
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v55
; SI-NEXT: s_lshr_b32 s29, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v7
-; SI-NEXT: v_readfirstlane_b32 s44, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
+; SI-NEXT: v_readfirstlane_b32 s6, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s44, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v23
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v39
+; SI-NEXT: v_readfirstlane_b32 s8, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
; SI-NEXT: v_readfirstlane_b32 s6, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_lshr_b32 s45, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v9
-; SI-NEXT: v_readfirstlane_b32 s58, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
+; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: v_readfirstlane_b32 s58, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_readfirstlane_b32 s6, v9
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v20
+; SI-NEXT: v_readfirstlane_b32 s8, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v60
+; SI-NEXT: v_readfirstlane_b32 s6, v11
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_lshr_b32 s59, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: v_readfirstlane_b32 s72, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v43
; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v11
+; SI-NEXT: v_readfirstlane_b32 s8, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v42
+; SI-NEXT: v_readfirstlane_b32 s6, v16
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v42
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: s_lshr_b32 s73, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v11
+; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v18
+; SI-NEXT: s_lshr_b64 s[92:93], s[8:9], 16
+; SI-NEXT: v_readfirstlane_b32 s6, v22
; SI-NEXT: s_lshr_b32 s79, s6, 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v53
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v51
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
+; SI-NEXT: s_mov_b32 s77, s74
; SI-NEXT: s_lshr_b64 s[54:55], s[58:59], 16
; SI-NEXT: s_mov_b32 s63, s54
; SI-NEXT: s_lshr_b64 s[60:61], s[44:45], 16
@@ -165221,155 +164775,149 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_mov_b32 s41, s42
; SI-NEXT: s_lshr_b64 s[34:35], s[22:23], 16
; SI-NEXT: s_mov_b32 s25, s34
-; SI-NEXT: v_readfirstlane_b32 s5, v14
-; SI-NEXT: s_lshr_b32 s13, s5, 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[18:19], 16
+; SI-NEXT: s_mov_b32 s17, s26
+; SI-NEXT: s_mov_b32 s11, s20
; SI-NEXT: s_lshr_b64 vcc, s[12:13], 16
; SI-NEXT: s_mov_b32 s5, vcc_lo
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v2
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v8
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v3
+; SI-NEXT: s_lshr_b32 s28, s74, 8
; SI-NEXT: s_lshr_b32 s22, s60, 8
; SI-NEXT: s_lshr_b32 s21, s42, 8
; SI-NEXT: s_lshr_b32 s18, s34, 8
; SI-NEXT: s_lshr_b32 s12, s20, 8
-; SI-NEXT: v_lshrrev_b32_e32 v18, 24, v18
-; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v10
-; SI-NEXT: v_lshrrev_b32_e32 v9, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v41, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v21, 24, v11
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v5, 24, v5
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v14
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s72, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[92:93], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
-; SI-NEXT: s_mov_b32 s77, s74
-; SI-NEXT: s_lshr_b32 s28, s74, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s78, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v15
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s78, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b64 s[48:49], s[78:79], 16
; SI-NEXT: s_mov_b32 s93, s48
; SI-NEXT: s_lshr_b32 s27, s48, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s6, v11
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s8, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_readfirstlane_b32 s6, v10
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[50:51], s[8:9], 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s56, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v41
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s6, v3
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v8
+; SI-NEXT: v_readfirstlane_b32 s56, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v59
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_readfirstlane_b32 s8, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_readfirstlane_b32 s6, v8
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: s_lshr_b32 s57, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v19
; SI-NEXT: s_lshr_b64 s[30:31], s[8:9], 16
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s6, v24
-; SI-NEXT: s_lshr_b32 s89, s6, 16
; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
; SI-NEXT: s_mov_b32 s51, s94
; SI-NEXT: s_lshr_b32 s44, s94, 8
; SI-NEXT: s_mov_b32 s56, s42
-; SI-NEXT: v_lshrrev_b32_e32 v47, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s88, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
-; SI-NEXT: s_mov_b32 s31, s82
-; SI-NEXT: s_lshr_b32 s43, s82, 8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v8
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v10
+; SI-NEXT: v_readfirstlane_b32 s88, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_readfirstlane_b32 s6, v13
+; SI-NEXT: v_readfirstlane_b32 s8, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_readfirstlane_b32 s36, v13
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: s_lshr_b32 s89, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v14
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[52:53], s[8:9], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: v_readfirstlane_b32 s6, v4
; SI-NEXT: s_lshr_b32 s37, s6, 16
-; SI-NEXT: s_mov_b32 s88, vcc_lo
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s36, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v59
; SI-NEXT: s_lshr_b64 s[98:99], s[36:37], 16
; SI-NEXT: s_mov_b32 s53, s98
+; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
+; SI-NEXT: s_mov_b32 s31, s82
; SI-NEXT: s_lshr_b32 s58, s98, 8
+; SI-NEXT: s_lshr_b32 s43, s82, 8
; SI-NEXT: s_mov_b32 s36, s26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b32 s88, vcc_lo
+; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v22
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_readfirstlane_b32 s6, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v11
-; SI-NEXT: v_readfirstlane_b32 s90, v13
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v60
+; SI-NEXT: v_readfirstlane_b32 s8, v13
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_readfirstlane_b32 s6, v14
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: s_lshr_b64 s[66:67], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v2
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_readfirstlane_b32 s90, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s8, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_readfirstlane_b32 s6, v2
; SI-NEXT: s_lshr_b32 s91, s6, 16
; SI-NEXT: s_lshr_b64 s[38:39], s[90:91], 16
; SI-NEXT: s_mov_b32 s67, s38
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; SI-NEXT: s_lshr_b32 s75, s38, 8
; SI-NEXT: s_mov_b32 s90, s74
; SI-NEXT: s_lshr_b32 s74, s54, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_readfirstlane_b32 s8, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v6
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v13
+; SI-NEXT: v_readfirstlane_b32 s68, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s6, v15
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[80:81], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v13
-; SI-NEXT: v_readfirstlane_b32 s68, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: s_lshr_b32 s69, s6, 16
@@ -165378,54 +164926,47 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b32 s72, s70, 8
; SI-NEXT: s_mov_b32 s68, s60
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s8, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_readfirstlane_b32 s8, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s64, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_readfirstlane_b32 s6, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[86:87], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v12
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: s_lshr_b32 s65, s6, 16
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_readfirstlane_b32 s64, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s8, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_readfirstlane_b32 s6, v17
+; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[84:85], s[64:65], 16
; SI-NEXT: s_mov_b32 s87, s84
-; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v6
; SI-NEXT: s_lshr_b32 s61, s84, 8
; SI-NEXT: s_mov_b32 s64, s48
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s8, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_readfirstlane_b32 s6, v6
; SI-NEXT: s_lshr_b32 s7, s6, 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v7
-; SI-NEXT: v_lshrrev_b32_e32 v16, 24, v12
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s6, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v14
; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
; SI-NEXT: s_mov_b32 s9, s96
; SI-NEXT: s_lshr_b64 s[14:15], s[8:9], 24
@@ -165571,33 +165112,29 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s15, 31
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 8
; SI-NEXT: v_writelane_b32 v61, s14, 28
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v5
; SI-NEXT: v_writelane_b32 v61, s15, 29
; SI-NEXT: s_lshr_b32 s78, s96, 8
; SI-NEXT: s_lshr_b32 s15, s26, 8
; SI-NEXT: s_mov_b32 s14, s20
; SI-NEXT: s_lshr_b32 s6, vcc_lo, 8
-; SI-NEXT: v_mov_b32_e32 v14, v4
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_lshrrev_b32_e32 v19, 24, v6
+; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v12
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v13
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: .LBB91_5: ; %end
; SI-NEXT: s_and_b32 s5, s8, 0xff
; SI-NEXT: v_readlane_b32 s8, v62, 0
-; SI-NEXT: v_readlane_b32 s9, v62, 1
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: v_readlane_b32 s8, v62, 2
-; SI-NEXT: v_readlane_b32 s9, v62, 3
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 4
+; SI-NEXT: v_readlane_b32 s9, v62, 4
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, vcc_lo, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_and_b32 s5, s96, 0xff
; SI-NEXT: s_lshl_b32 s8, s78, 8
@@ -165606,29 +165143,24 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v48
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v19
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 6
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 5
+; SI-NEXT: v_readlane_b32 s7, v62, 6
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s86, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 7
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 8
-; SI-NEXT: v_readlane_b32 s9, v62, 9
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 10
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 10
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, vcc_lo, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 11
; SI-NEXT: v_mov_b32_e32 v2, s5
; SI-NEXT: s_and_b32 s5, s84, 0xff
; SI-NEXT: s_lshl_b32 s8, s61, 8
@@ -165638,87 +165170,109 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s65, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v14
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 12
+; SI-NEXT: v_readlane_b32 s7, v62, 12
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s80, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 13
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 14
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 14
-; SI-NEXT: v_readlane_b32 s9, v62, 15
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 16
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 16
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 12, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s70, 0xff
; SI-NEXT: s_lshl_b32 s8, s72, 8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s69, 0xff
-; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v15
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v62, 18
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 20, v0
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: v_readlane_b32 s99, v63, 35
+; SI-NEXT: v_readlane_b32 s97, v63, 33
+; SI-NEXT: v_readlane_b32 s96, v63, 32
+; SI-NEXT: v_readlane_b32 s87, v63, 31
+; SI-NEXT: v_readlane_b32 s86, v63, 30
+; SI-NEXT: v_readlane_b32 s85, v63, 29
+; SI-NEXT: v_readlane_b32 s84, v63, 28
+; SI-NEXT: v_readlane_b32 s83, v63, 27
+; SI-NEXT: v_readlane_b32 s81, v63, 25
+; SI-NEXT: v_readlane_b32 s80, v63, 24
+; SI-NEXT: v_readlane_b32 s71, v63, 23
+; SI-NEXT: v_readlane_b32 s70, v63, 22
+; SI-NEXT: v_readlane_b32 s69, v63, 21
+; SI-NEXT: v_readlane_b32 s67, v63, 19
+; SI-NEXT: v_readlane_b32 s65, v63, 17
+; SI-NEXT: v_readlane_b32 s55, v63, 15
+; SI-NEXT: v_readlane_b32 s53, v63, 13
+; SI-NEXT: v_readlane_b32 s51, v63, 11
+; SI-NEXT: v_readlane_b32 s49, v63, 9
+; SI-NEXT: v_readlane_b32 s48, v63, 8
+; SI-NEXT: v_readlane_b32 s39, v63, 7
+; SI-NEXT: v_readlane_b32 s35, v63, 3
+; SI-NEXT: v_readlane_b32 s31, v63, 1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 18
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s66, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 19
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 17
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 20
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 20
-; SI-NEXT: v_readlane_b32 s9, v62, 21
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 22
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 22
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 20, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s38, 0xff
; SI-NEXT: s_lshl_b32 s8, s75, 8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s91, 0xff
-; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v13
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v62, 24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 28, v0
+; SI-NEXT: v_readlane_b32 s66, v63, 18
+; SI-NEXT: v_readlane_b32 s38, v63, 6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 24
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s52, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 25
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 23
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 26
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 26
-; SI-NEXT: v_readlane_b32 s9, v62, 27
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 28
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 28
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 28, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
@@ -165730,22 +165284,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s37, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v11
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v41
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 30
+; SI-NEXT: v_readlane_b32 s7, v62, 30
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s30, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 31
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 29
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 32
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 32
-; SI-NEXT: v_readlane_b32 s9, v62, 33
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 34
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 34
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165761,21 +165312,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s89, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v47
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v31
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 36
+; SI-NEXT: v_readlane_b32 s7, v62, 36
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s50, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 37
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 38
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 38
-; SI-NEXT: v_readlane_b32 s9, v62, 39
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s42, v62, 40
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 40
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s42, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165791,22 +165340,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s57, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v45
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 42
+; SI-NEXT: v_readlane_b32 s7, v62, 42
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s92, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 43
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s43, v62, 41
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 44
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 44
-; SI-NEXT: v_readlane_b32 s9, v62, 45
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s42, v62, 46
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 46
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s42, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165822,56 +165368,60 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s79, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v18
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v10
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 48
+; SI-NEXT: v_readlane_b32 s7, v62, 48
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s76, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 49
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 50
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 50
-; SI-NEXT: v_readlane_b32 s9, v62, 51
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 52
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 52
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 60, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s90, 0xff
; SI-NEXT: s_lshl_b32 s8, s28, 8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s73, 0xff
-; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v55
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v62, 54
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 0x44, v0
+; SI-NEXT: v_readlane_b32 s98, v63, 34
+; SI-NEXT: v_readlane_b32 s82, v63, 26
+; SI-NEXT: v_readlane_b32 s64, v63, 16
+; SI-NEXT: v_readlane_b32 s52, v63, 12
+; SI-NEXT: v_readlane_b32 s50, v63, 10
+; SI-NEXT: v_readlane_b32 s37, v63, 5
+; SI-NEXT: v_readlane_b32 s30, v63, 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 54
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s62, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 55
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v62, 53
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 56
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 56
-; SI-NEXT: v_readlane_b32 s9, v62, 57
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 58
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 58
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x44, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
@@ -165883,22 +165433,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s59, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v9
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v21
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 60
+; SI-NEXT: v_readlane_b32 s7, v62, 60
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s46, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 61
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v62, 59
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 62
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 62
-; SI-NEXT: v_readlane_b32 s9, v62, 63
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v61, 0
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 0
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165914,22 +165461,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s45, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v7
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 2
+; SI-NEXT: v_readlane_b32 s7, v61, 2
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s40, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 3
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v61, 1
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 4
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 4
-; SI-NEXT: v_readlane_b32 s9, v61, 5
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v61, 6
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 6
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165945,127 +165489,85 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s29, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v14
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v5
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 8
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s24, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 9
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 10
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 10
-; SI-NEXT: v_readlane_b32 s9, v61, 11
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 12
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 12
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x5c, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x60, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s34, 0xff
; SI-NEXT: s_lshl_b32 s8, s18, 8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x60, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s23, 0xff
-; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v56
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v61, 14
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 0x64, v0
+; SI-NEXT: v_readlane_b32 s68, v63, 20
+; SI-NEXT: v_readlane_b32 s54, v63, 14
+; SI-NEXT: v_readlane_b32 s34, v63, 2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 14
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s16, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 15
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 16
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 16
-; SI-NEXT: v_readlane_b32 s9, v61, 17
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s16, v61, 18
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 18
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s16, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x64, v0
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x68, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s36, 0xff
; SI-NEXT: s_lshl_b32 s8, s15, 8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s19, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v61, 20
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x6c, v0
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s61, v62, 35
-; SI-NEXT: v_readlane_b32 s43, v62, 47
-; SI-NEXT: v_readlane_b32 s27, v61, 7
-; SI-NEXT: v_readlane_b32 s21, v61, 13
-; SI-NEXT: v_readlane_b32 s17, v61, 19
-; SI-NEXT: v_readlane_b32 s99, v63, 35
-; SI-NEXT: v_readlane_b32 s98, v63, 34
-; SI-NEXT: v_readlane_b32 s97, v63, 33
-; SI-NEXT: v_readlane_b32 s96, v63, 32
-; SI-NEXT: v_readlane_b32 s87, v63, 31
-; SI-NEXT: v_readlane_b32 s86, v63, 30
-; SI-NEXT: v_readlane_b32 s85, v63, 29
-; SI-NEXT: v_readlane_b32 s84, v63, 28
-; SI-NEXT: v_readlane_b32 s83, v63, 27
-; SI-NEXT: v_readlane_b32 s82, v63, 26
-; SI-NEXT: v_readlane_b32 s81, v63, 25
-; SI-NEXT: v_readlane_b32 s80, v63, 24
-; SI-NEXT: v_readlane_b32 s71, v63, 23
-; SI-NEXT: v_readlane_b32 s70, v63, 22
-; SI-NEXT: v_readlane_b32 s69, v63, 21
-; SI-NEXT: v_readlane_b32 s68, v63, 20
-; SI-NEXT: v_readlane_b32 s67, v63, 19
-; SI-NEXT: v_readlane_b32 s66, v63, 18
-; SI-NEXT: v_readlane_b32 s65, v63, 17
-; SI-NEXT: v_readlane_b32 s64, v63, 16
-; SI-NEXT: v_readlane_b32 s55, v63, 15
-; SI-NEXT: v_readlane_b32 s54, v63, 14
-; SI-NEXT: v_readlane_b32 s53, v63, 13
-; SI-NEXT: v_readlane_b32 s52, v63, 12
-; SI-NEXT: v_readlane_b32 s51, v63, 11
-; SI-NEXT: v_readlane_b32 s50, v63, 10
-; SI-NEXT: v_readlane_b32 s49, v63, 9
-; SI-NEXT: v_readlane_b32 s48, v63, 8
-; SI-NEXT: v_readlane_b32 s39, v63, 7
-; SI-NEXT: v_readlane_b32 s38, v63, 6
-; SI-NEXT: v_readlane_b32 s37, v63, 5
; SI-NEXT: v_readlane_b32 s36, v63, 4
-; SI-NEXT: v_readlane_b32 s35, v63, 3
-; SI-NEXT: v_readlane_b32 s34, v63, 2
-; SI-NEXT: v_readlane_b32 s31, v63, 1
-; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 20
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s10, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 21
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 22
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 22
-; SI-NEXT: v_readlane_b32 s9, v61, 23
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v61, 24
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -166074,29 +165576,25 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x70, v0
; SI-NEXT: v_mov_b32_e32 v2, s5
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s5, s14, 0xff
; SI-NEXT: s_lshl_b32 s8, s12, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 27
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 26
-; SI-NEXT: v_readlane_b32 s9, v61, 27
-; SI-NEXT: s_and_b32 s8, s9, 0xff
+; SI-NEXT: s_and_b32 s8, s7, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x74, v0
-; SI-NEXT: v_readlane_b32 s11, v61, 25
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 28
-; SI-NEXT: v_readlane_b32 s9, v61, 29
; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v61, 30
-; SI-NEXT: v_readlane_b32 s9, v61, 31
+; SI-NEXT: v_readlane_b32 s5, v61, 28
+; SI-NEXT: s_lshl_b32 s5, s5, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s8, 0xff
+; SI-NEXT: v_readlane_b32 s5, v61, 30
+; SI-NEXT: s_and_b32 s5, s5, 0xff
; SI-NEXT: v_readlane_b32 s8, v61, 32
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
@@ -166104,23 +165602,20 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s8, s5
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x78, v0
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s88, 0xff
; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x78, v0
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s13, 0xff
+; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v3
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
-; SI-NEXT: v_readlane_b32 s9, v61, 33
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
@@ -167620,9 +167115,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: s_lshl_b32 s7, s7, 8
; VI-NEXT: s_or_b32 s5, s5, s7
; VI-NEXT: v_readlane_b32 s7, v22, 63
-; VI-NEXT: v_readlane_b32 s12, v22, 0
+; VI-NEXT: v_readlane_b32 s9, v22, 0
; VI-NEXT: s_and_b32 s7, s7, 0xff
-; VI-NEXT: s_lshl_b32 s9, s12, 8
+; VI-NEXT: s_lshl_b32 s9, s9, 8
; VI-NEXT: s_or_b32 s7, s7, s9
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_lshl_b32 s7, s7, 16
@@ -167650,9 +167145,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: s_lshl_b32 s7, s7, 8
; VI-NEXT: s_or_b32 s5, s5, s7
; VI-NEXT: v_readlane_b32 s7, v22, 58
-; VI-NEXT: v_readlane_b32 s10, v22, 2
+; VI-NEXT: v_readlane_b32 s9, v22, 2
; VI-NEXT: s_and_b32 s7, s7, 0xff
-; VI-NEXT: s_lshl_b32 s9, s10, 8
+; VI-NEXT: s_lshl_b32 s9, s9, 8
; VI-NEXT: s_or_b32 s7, s7, s9
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_lshl_b32 s7, s7, 16
@@ -167707,13 +167202,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: s_and_b32 s5, s6, 0xff
; VI-NEXT: v_readlane_b32 s6, v22, 49
-; VI-NEXT: v_readlane_b32 s9, v22, 5
; VI-NEXT: s_lshl_b32 s6, s6, 8
; VI-NEXT: s_or_b32 s5, s5, s6
; VI-NEXT: v_readlane_b32 s6, v22, 48
-; VI-NEXT: v_readlane_b32 s8, v22, 6
+; VI-NEXT: v_readlane_b32 s7, v22, 6
; VI-NEXT: s_and_b32 s6, s6, 0xff
-; VI-NEXT: s_lshl_b32 s7, s8, 8
+; VI-NEXT: s_lshl_b32 s7, s7, 8
; VI-NEXT: s_or_b32 s6, s6, s7
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_lshl_b32 s6, s6, 16
@@ -167767,10 +167261,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_readlane_b32 s13, v22, 1
-; VI-NEXT: v_readlane_b32 s11, v22, 3
-; VI-NEXT: v_readlane_b32 s9, v22, 7
-; VI-NEXT: v_readlane_b32 s7, v22, 9
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s87, v20, 31
; VI-NEXT: v_readlane_b32 s86, v20, 30
@@ -171042,9 +170532,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: s_and_b32 s16, s16, 0xff
; GFX11-NEXT: s_lshl_b32 s19, s73, 8
; GFX11-NEXT: s_or_b32 s16, s16, s17
-; GFX11-NEXT: v_readlane_b32 s96, v19, 0
+; GFX11-NEXT: s_lshl_b32 s17, s70, 8
; GFX11-NEXT: s_lshl_b32 s16, s16, 16
-; GFX11-NEXT: v_readlane_b32 s81, v18, 25
+; GFX11-NEXT: v_readlane_b32 s96, v19, 0
; GFX11-NEXT: s_or_b32 s3, s3, s16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v8, s3
@@ -171053,20 +170543,20 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: s_or_b32 s1, s2, s1
; GFX11-NEXT: s_lshl_b32 s2, s59, 8
; GFX11-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-NEXT: v_readlane_b32 s16, v20, 0
+; GFX11-NEXT: s_and_b32 s16, s64, 0xff
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: s_and_b32 s1, s21, 0xff
-; GFX11-NEXT: v_readlane_b32 s17, v20, 1
-; GFX11-NEXT: s_or_b32 s1, s1, s2
-; GFX11-NEXT: v_readlane_b32 s2, v20, 18
-; GFX11-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX11-NEXT: s_lshl_b32 s17, s70, 8
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off
; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
+; GFX11-NEXT: s_or_b32 s1, s1, s2
+; GFX11-NEXT: v_readlane_b32 s2, v20, 18
+; GFX11-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-NEXT: v_readlane_b32 s81, v18, 25
; GFX11-NEXT: v_readlane_b32 s70, v18, 22
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
; GFX11-NEXT: v_readlane_b32 s69, v18, 21
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: v_readlane_b32 s64, v18, 16
; GFX11-NEXT: s_or_b32 s2, s2, s3
; GFX11-NEXT: s_and_b32 s3, s22, 0xff
; GFX11-NEXT: s_lshl_b32 s2, s2, 16
@@ -171075,9 +170565,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: s_lshl_b32 s2, s80, 8
; GFX11-NEXT: v_dual_mov_b32 v9, s0 :: v_dual_mov_b32 v10, s1
; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_lshl_b32 s3, s16, 8
-; GFX11-NEXT: s_and_b32 s16, s64, 0xff
+; GFX11-NEXT: v_readlane_b32 s3, v20, 0
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX11-NEXT: s_and_b32 s0, s24, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s49, 8
+; GFX11-NEXT: v_readlane_b32 s80, v18, 24
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: s_or_b32 s3, s16, s3
; GFX11-NEXT: s_and_b32 s16, s23, 0xff
; GFX11-NEXT: s_lshl_b32 s3, s3, 16
@@ -171085,29 +170579,29 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: v_readlane_b32 s17, v20, 19
; GFX11-NEXT: s_or_b32 s2, s2, s3
; GFX11-NEXT: s_and_b32 s3, s16, 0xffff
-; GFX11-NEXT: s_and_b32 s0, s24, 0xff
-; GFX11-NEXT: s_lshl_b32 s1, s49, 8
+; GFX11-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-NEXT: v_readlane_b32 s49, v18, 9
; GFX11-NEXT: s_and_b32 s17, s17, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s17, s17, s18
-; GFX11-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX11-NEXT: s_lshl_b32 s16, s17, 16
; GFX11-NEXT: s_and_b32 s18, s71, 0xff
+; GFX11-NEXT: s_lshl_b32 s16, s17, 16
+; GFX11-NEXT: s_lshl_b32 s17, s66, 8
; GFX11-NEXT: s_or_b32 s3, s3, s16
-; GFX11-NEXT: v_readlane_b32 s16, v20, 2
; GFX11-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
-; GFX11-NEXT: s_and_b32 s2, s68, 0xff
-; GFX11-NEXT: v_readlane_b32 s17, v20, 3
-; GFX11-NEXT: s_lshl_b32 s3, s16, 8
+; GFX11-NEXT: v_readlane_b32 s3, v20, 2
; GFX11-NEXT: v_readlane_b32 s16, v20, 20
+; GFX11-NEXT: s_and_b32 s2, s68, 0xff
+; GFX11-NEXT: v_readlane_b32 s71, v18, 23
+; GFX11-NEXT: v_readlane_b32 s68, v18, 20
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_and_b32 s16, s16, 0xff
; GFX11-NEXT: s_or_b32 s1, s2, s3
; GFX11-NEXT: s_and_b32 s2, s25, 0xff
; GFX11-NEXT: s_lshl_b32 s3, s67, 8
-; GFX11-NEXT: s_lshl_b32 s17, s66, 8
-; GFX11-NEXT: s_and_b32 s16, s16, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-NEXT: s_or_b32 s2, s2, s3
; GFX11-NEXT: s_or_b32 s3, s16, s17
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
; GFX11-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-NEXT: s_or_b32 s0, s0, s1
@@ -171254,14 +170748,15 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: s_lshl_b32 s9, s9, 16
; GFX11-NEXT: s_or_b32 s2, s2, s3
; GFX11-NEXT: s_or_b32 s3, s8, s9
-; GFX11-NEXT: v_readlane_b32 s8, v20, 4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s3
+; GFX11-NEXT: v_readlane_b32 s3, v20, 4
; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
; GFX11-NEXT: s_and_b32 s0, s6, 0xff
; GFX11-NEXT: v_readlane_b32 s6, v20, 12
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s3
; GFX11-NEXT: s_lshl_b32 s1, vcc_hi, 8
; GFX11-NEXT: s_and_b32 s2, s95, 0xff
-; GFX11-NEXT: s_lshl_b32 s3, s8, 8
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: s_or_b32 s1, s2, s3
; GFX11-NEXT: s_and_b32 s2, s7, 0xff
@@ -171281,7 +170776,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: s_lshl_b32 s3, s75, 8
; GFX11-NEXT: s_and_b32 s4, s63, 0xff
; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: v_readlane_b32 s7, v20, 7
; GFX11-NEXT: s_or_b32 s2, s2, s3
; GFX11-NEXT: s_or_b32 s3, s4, s6
; GFX11-NEXT: s_and_b32 s4, s5, 0xff
@@ -171302,7 +170796,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:64
; GFX11-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_mov_b32 v6, s1
; GFX11-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v8, s3
-; GFX11-NEXT: v_readlane_b32 s9, v20, 5
; GFX11-NEXT: s_clause 0x2
; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:80
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:96
@@ -171315,18 +170808,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-NEXT: v_readlane_b32 s84, v18, 28
; GFX11-NEXT: v_readlane_b32 s83, v18, 27
; GFX11-NEXT: v_readlane_b32 s82, v18, 26
-; GFX11-NEXT: v_readlane_b32 s80, v18, 24
-; GFX11-NEXT: v_readlane_b32 s71, v18, 23
-; GFX11-NEXT: v_readlane_b32 s68, v18, 20
; GFX11-NEXT: v_readlane_b32 s67, v18, 19
; GFX11-NEXT: v_readlane_b32 s66, v18, 18
; GFX11-NEXT: v_readlane_b32 s65, v18, 17
-; GFX11-NEXT: v_readlane_b32 s64, v18, 16
; GFX11-NEXT: v_readlane_b32 s55, v18, 15
; GFX11-NEXT: v_readlane_b32 s54, v18, 14
; GFX11-NEXT: v_readlane_b32 s52, v18, 12
; GFX11-NEXT: v_readlane_b32 s51, v18, 11
-; GFX11-NEXT: v_readlane_b32 s49, v18, 9
; GFX11-NEXT: v_readlane_b32 s48, v18, 8
; GFX11-NEXT: v_readlane_b32 s39, v18, 7
; GFX11-NEXT: v_readlane_b32 s38, v18, 6
@@ -185833,9 +185321,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 4, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v27
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; VI-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
@@ -185876,15 +185363,13 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 20, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v23
-; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v1, vcc, 24, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
@@ -185898,15 +185383,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 28, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT: v_or_b32_sdwa v0, v19, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: v_or_b32_sdwa v0, v19, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -185922,13 +185406,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 36, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT: v_or_b32_sdwa v0, v14, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: v_or_b32_sdwa v0, v14, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v43, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v1, vcc, 40, v12
@@ -185943,15 +185426,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 44, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -185967,11 +185449,10 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: v_add_u32_e32 v1, vcc, 52, v12
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -186118,16 +185599,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v3
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x70, v12
@@ -186145,13 +185624,13 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -186929,9 +186408,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -186948,9 +186426,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -186992,12 +186469,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -187018,12 +186495,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -187044,12 +186521,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -187070,12 +186547,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -187096,12 +186573,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -187122,12 +186599,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -188321,8 +187798,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
@@ -188351,185 +187828,189 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:36
; SI-NEXT: s_waitcnt expcnt(4)
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:32
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:44
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:44
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:48
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:64
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:72
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:64
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:72
; SI-NEXT: v_writelane_b32 v63, s30, 0
+; SI-NEXT: v_mov_b32_e32 v31, v30
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v8
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_writelane_b32 v63, s34, 2
; SI-NEXT: v_writelane_b32 v63, s35, 3
; SI-NEXT: v_writelane_b32 v63, s36, 4
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v7
; SI-NEXT: v_writelane_b32 v63, s37, 5
; SI-NEXT: v_writelane_b32 v63, s38, 6
; SI-NEXT: v_writelane_b32 v63, s39, 7
; SI-NEXT: v_writelane_b32 v63, s48, 8
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v9
; SI-NEXT: v_writelane_b32 v63, s49, 9
; SI-NEXT: v_writelane_b32 v63, s50, 10
; SI-NEXT: v_writelane_b32 v63, s51, 11
; SI-NEXT: v_writelane_b32 v63, s52, 12
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v9
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v14
; SI-NEXT: v_writelane_b32 v63, s53, 13
; SI-NEXT: v_writelane_b32 v63, s54, 14
; SI-NEXT: v_writelane_b32 v63, s55, 15
+; SI-NEXT: v_mov_b32_e32 v44, v18
; SI-NEXT: v_writelane_b32 v63, s64, 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v44
; SI-NEXT: v_writelane_b32 v63, s65, 17
; SI-NEXT: v_writelane_b32 v63, s66, 18
; SI-NEXT: v_writelane_b32 v63, s67, 19
; SI-NEXT: v_writelane_b32 v63, s68, 20
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v25
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v22
; SI-NEXT: v_writelane_b32 v63, s69, 21
; SI-NEXT: v_writelane_b32 v63, s70, 22
; SI-NEXT: v_writelane_b32 v63, s71, 23
+; SI-NEXT: v_mov_b32_e32 v33, v26
; SI-NEXT: v_writelane_b32 v63, s80, 24
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v30
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v33
; SI-NEXT: v_writelane_b32 v63, s81, 25
; SI-NEXT: v_writelane_b32 v63, s82, 26
; SI-NEXT: v_writelane_b32 v63, s83, 27
; SI-NEXT: v_writelane_b32 v63, s84, 28
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v31
; SI-NEXT: v_writelane_b32 v63, s85, 29
; SI-NEXT: v_writelane_b32 v63, s86, 30
; SI-NEXT: v_writelane_b32 v63, s87, 31
-; SI-NEXT: v_mov_b32_e32 v46, v29
; SI-NEXT: v_writelane_b32 v63, s96, 32
-; SI-NEXT: v_cvt_f16_f32_e32 v47, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v33, v6
-; SI-NEXT: v_cvt_f16_f32_e32 v43, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v12
-; SI-NEXT: v_cvt_f16_f32_e32 v29, v11
-; SI-NEXT: v_cvt_f16_f32_e32 v31, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v13
-; SI-NEXT: v_cvt_f16_f32_e32 v58, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v15
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v43, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_cvt_f16_f32_e32 v42, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v8, v10
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v26, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v7, v13
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v34
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v18
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v17
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v37
-; SI-NEXT: v_cvt_f16_f32_e32 v34, v20
-; SI-NEXT: v_cvt_f16_f32_e32 v16, v19
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v22
-; SI-NEXT: v_cvt_f16_f32_e32 v61, v21
-; SI-NEXT: v_cvt_f16_f32_e32 v21, v24
-; SI-NEXT: v_cvt_f16_f32_e32 v24, v23
-; SI-NEXT: v_cvt_f16_f32_e32 v44, v28
-; SI-NEXT: v_cvt_f16_f32_e32 v42, v27
-; SI-NEXT: v_cvt_f16_f32_e32 v46, v46
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v35
-; SI-NEXT: v_cvt_f16_f32_e32 v23, v36
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v38
-; SI-NEXT: v_cvt_f16_f32_e32 v27, v39
-; SI-NEXT: v_cvt_f16_f32_e32 v28, v48
+; SI-NEXT: v_cvt_f16_f32_e32 v18, v16
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v59, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v20
+; SI-NEXT: v_cvt_f16_f32_e32 v41, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v34, v21
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v9, v23
+; SI-NEXT: v_cvt_f16_f32_e32 v47, v25
+; SI-NEXT: v_cvt_f16_f32_e32 v25, v28
+; SI-NEXT: v_cvt_f16_f32_e32 v44, v27
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v29
+; SI-NEXT: v_cvt_f16_f32_e32 v46, v35
+; SI-NEXT: v_cvt_f16_f32_e32 v33, v36
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v37
+; SI-NEXT: v_cvt_f16_f32_e32 v36, v38
+; SI-NEXT: v_cvt_f16_f32_e32 v31, v39
+; SI-NEXT: v_cvt_f16_f32_e32 v32, v48
; SI-NEXT: v_cvt_f16_f32_e32 v4, v49
-; SI-NEXT: v_cvt_f16_f32_e32 v45, v45
-; SI-NEXT: v_cvt_f16_f32_e32 v36, v56
-; SI-NEXT: v_cvt_f16_f32_e32 v57, v57
-; SI-NEXT: v_cvt_f16_f32_e32 v59, v59
-; SI-NEXT: v_cvt_f16_f32_e32 v56, v60
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v45
+; SI-NEXT: v_cvt_f16_f32_e32 v45, v56
+; SI-NEXT: v_cvt_f16_f32_e32 v56, v57
+; SI-NEXT: v_cvt_f16_f32_e32 v38, v58
+; SI-NEXT: v_cvt_f16_f32_e32 v57, v60
+; SI-NEXT: v_cvt_f16_f32_e32 v61, v61
+; SI-NEXT: v_cvt_f16_f32_e32 v58, v50
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cvt_f16_f32_e32 v26, v50
+; SI-NEXT: v_cvt_f16_f32_e32 v51, v51
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_cvt_f16_f32_e32 v25, v51
+; SI-NEXT: v_cvt_f16_f32_e32 v60, v52
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_cvt_f16_f32_e32 v52, v52
+; SI-NEXT: v_cvt_f16_f32_e32 v37, v53
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_cvt_f16_f32_e32 v50, v53
+; SI-NEXT: v_cvt_f16_f32_e32 v53, v54
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_cvt_f16_f32_e32 v49, v54
-; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_cvt_f16_f32_e32 v54, v55
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_cvt_f16_f32_e32 v53, v40
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_cvt_f16_f32_e32 v55, v41
-; SI-NEXT: v_cvt_f16_f32_e32 v19, s17
-; SI-NEXT: v_cvt_f16_f32_e32 v18, s16
-; SI-NEXT: v_cvt_f16_f32_e32 v12, s19
-; SI-NEXT: v_cvt_f16_f32_e32 v17, s18
-; SI-NEXT: v_cvt_f16_f32_e32 v38, s21
-; SI-NEXT: v_cvt_f16_f32_e32 v37, s20
-; SI-NEXT: v_cvt_f16_f32_e32 v48, s23
-; SI-NEXT: v_cvt_f16_f32_e32 v22, s22
-; SI-NEXT: v_cvt_f16_f32_e32 v15, s25
-; SI-NEXT: v_cvt_f16_f32_e32 v14, s24
-; SI-NEXT: v_cvt_f16_f32_e32 v30, s27
-; SI-NEXT: v_cvt_f16_f32_e32 v39, s26
-; SI-NEXT: v_cvt_f16_f32_e32 v35, s29
-; SI-NEXT: v_cvt_f16_f32_e32 v20, s28
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_cvt_f16_f32_e32 v55, v40
+; SI-NEXT: v_cvt_f16_f32_e32 v22, s17
+; SI-NEXT: v_cvt_f16_f32_e32 v20, s16
+; SI-NEXT: v_cvt_f16_f32_e32 v40, s19
+; SI-NEXT: v_cvt_f16_f32_e32 v19, s18
+; SI-NEXT: v_cvt_f16_f32_e32 v28, s21
+; SI-NEXT: v_cvt_f16_f32_e32 v27, s20
+; SI-NEXT: v_cvt_f16_f32_e32 v13, s23
+; SI-NEXT: v_cvt_f16_f32_e32 v24, s22
+; SI-NEXT: v_cvt_f16_f32_e32 v17, s25
+; SI-NEXT: v_cvt_f16_f32_e32 v16, s24
+; SI-NEXT: v_cvt_f16_f32_e32 v29, s27
+; SI-NEXT: v_cvt_f16_f32_e32 v15, s26
+; SI-NEXT: v_cvt_f16_f32_e32 v23, s29
+; SI-NEXT: v_cvt_f16_f32_e32 v21, s28
; SI-NEXT: v_writelane_b32 v63, s97, 33
; SI-NEXT: v_writelane_b32 v63, s98, 34
; SI-NEXT: v_writelane_b32 v63, s99, 35
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readfirstlane_b32 s4, v19
+; SI-NEXT: v_readfirstlane_b32 s4, v22
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: v_readfirstlane_b32 s5, v20
; SI-NEXT: s_or_b32 s44, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v12
+; SI-NEXT: v_readfirstlane_b32 s4, v40
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v17
+; SI-NEXT: v_readfirstlane_b32 s5, v19
; SI-NEXT: s_or_b32 s45, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 24
; SI-NEXT: v_writelane_b32 v62, s4, 4
@@ -188540,13 +188021,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 8
; SI-NEXT: v_writelane_b32 v62, s4, 0
; SI-NEXT: v_writelane_b32 v62, s5, 1
-; SI-NEXT: v_readfirstlane_b32 s4, v38
+; SI-NEXT: v_readfirstlane_b32 s4, v28
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v37
+; SI-NEXT: v_readfirstlane_b32 s5, v27
; SI-NEXT: s_or_b32 s42, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v48
+; SI-NEXT: v_readfirstlane_b32 s4, v13
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v22
+; SI-NEXT: v_readfirstlane_b32 s5, v24
; SI-NEXT: s_or_b32 s43, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 24
; SI-NEXT: v_writelane_b32 v62, s4, 10
@@ -188557,14 +188038,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 8
; SI-NEXT: v_writelane_b32 v62, s4, 6
; SI-NEXT: v_writelane_b32 v62, s5, 7
-; SI-NEXT: v_readfirstlane_b32 s5, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s4, v15
+; SI-NEXT: v_readfirstlane_b32 s5, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s4, v17
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s40, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v30
+; SI-NEXT: v_readfirstlane_b32 s4, v29
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v39
+; SI-NEXT: v_readfirstlane_b32 s5, v15
; SI-NEXT: s_or_b32 s41, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[40:41], 24
; SI-NEXT: v_writelane_b32 v62, s4, 16
@@ -188575,32 +188056,37 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[40:41], 8
; SI-NEXT: v_writelane_b32 v62, s4, 12
; SI-NEXT: v_writelane_b32 v62, s5, 13
-; SI-NEXT: v_readfirstlane_b32 s4, v35
+; SI-NEXT: v_readfirstlane_b32 s4, v23
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v20
+; SI-NEXT: v_readfirstlane_b32 s5, v21
+; SI-NEXT: v_mov_b32_e32 v19, v30
; SI-NEXT: s_or_b32 s28, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v47
+; SI-NEXT: v_readfirstlane_b32 s4, v19
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_mov_b32_e32 v22, v2
-; SI-NEXT: v_mov_b32_e32 v39, v5
-; SI-NEXT: v_mov_b32_e32 v60, v16
+; SI-NEXT: v_mov_b32_e32 v21, v6
+; SI-NEXT: v_mov_b32_e32 v6, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v22, v1
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v20, v43
; SI-NEXT: v_readfirstlane_b32 s46, v55
-; SI-NEXT: v_mov_b32_e32 v17, v43
-; SI-NEXT: v_mov_b32_e32 v40, v34
-; SI-NEXT: v_mov_b32_e32 v41, v21
-; SI-NEXT: v_mov_b32_e32 v51, v42
+; SI-NEXT: v_mov_b32_e32 v35, v37
+; SI-NEXT: v_mov_b32_e32 v49, v11
+; SI-NEXT: v_mov_b32_e32 v48, v9
+; SI-NEXT: v_mov_b32_e32 v39, v47
+; SI-NEXT: v_mov_b32_e32 v50, v2
+; SI-NEXT: v_mov_b32_e32 v52, v45
; SI-NEXT: s_lshr_b32 s71, s45, 8
; SI-NEXT: s_lshr_b32 s70, s43, 8
; SI-NEXT: s_lshr_b32 s69, s41, 8
-; SI-NEXT: v_bfe_u32 v38, v47, 8, 8
-; SI-NEXT: v_bfe_u32 v37, v33, 8, 8
-; SI-NEXT: v_bfe_u32 v35, v32, 8, 8
-; SI-NEXT: v_bfe_u32 v20, v10, 8, 8
-; SI-NEXT: v_bfe_u32 v19, v9, 8, 8
+; SI-NEXT: v_bfe_u32 v24, v21, 8, 8
+; SI-NEXT: v_mov_b32_e32 v28, v6
+; SI-NEXT: v_bfe_u32 v23, v6, 8, 8
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v55, v4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s5, v14
+; SI-NEXT: v_mov_b32_e32 v55, v5
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_readfirstlane_b32 s5, v16
; SI-NEXT: s_or_b32 s29, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[28:29], 24
; SI-NEXT: v_writelane_b32 v62, s4, 22
@@ -188611,15 +188097,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[28:29], 8
; SI-NEXT: v_writelane_b32 v62, s4, 18
; SI-NEXT: v_writelane_b32 v62, s5, 19
-; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_mov_b32_e32 v2, v1
-; SI-NEXT: v_readfirstlane_b32 s5, v1
+; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v43
; SI-NEXT: s_or_b32 s26, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v33
+; SI-NEXT: v_readfirstlane_b32 s4, v21
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v43
+; SI-NEXT: v_readfirstlane_b32 s5, v42
; SI-NEXT: s_or_b32 s27, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[26:27], 24
; SI-NEXT: v_writelane_b32 v62, s4, 28
@@ -188629,21 +188114,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v62, s5, 27
; SI-NEXT: s_lshr_b64 s[4:5], s[26:27], 8
; SI-NEXT: v_writelane_b32 v62, s4, 24
+; SI-NEXT: v_mov_b32_e32 v43, v42
; SI-NEXT: v_writelane_b32 v62, s5, 25
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s68, s29, 8
; SI-NEXT: s_lshr_b32 s66, s27, 8
-; SI-NEXT: v_bfe_u32 v43, v31, 8, 8
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s5, v1
+; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v15
; SI-NEXT: s_or_b32 s24, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v32
+; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_bfe_u32 v15, v5, 8, 8
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_or_b32 s25, s5, s4
@@ -188654,15 +188137,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v62, s4, 32
; SI-NEXT: v_writelane_b32 v62, s5, 33
; SI-NEXT: s_lshr_b64 s[4:5], s[24:25], 8
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 30
; SI-NEXT: v_writelane_b32 v62, s5, 31
-; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v29
+; SI-NEXT: v_readfirstlane_b32 s5, v26
+; SI-NEXT: v_mov_b32_e32 v12, v26
; SI-NEXT: s_or_b32 s22, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v31
+; SI-NEXT: v_mov_b32_e32 v26, v7
+; SI-NEXT: v_readfirstlane_b32 s5, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s4, v17
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v6
; SI-NEXT: s_or_b32 s23, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 24
; SI-NEXT: v_writelane_b32 v62, s4, 40
@@ -188673,107 +188160,95 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 8
; SI-NEXT: v_writelane_b32 v62, s4, 36
; SI-NEXT: v_writelane_b32 v62, s5, 37
-; SI-NEXT: v_readfirstlane_b32 s4, v58
+; SI-NEXT: v_readfirstlane_b32 s4, v18
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v14
; SI-NEXT: s_or_b32 s20, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: v_readfirstlane_b32 s4, v42
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v11
+; SI-NEXT: v_readfirstlane_b32 s5, v59
; SI-NEXT: s_or_b32 s21, s5, s4
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 24
; SI-NEXT: v_writelane_b32 v62, s4, 44
; SI-NEXT: v_writelane_b32 v62, s5, 45
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v58, v11
; SI-NEXT: v_writelane_b32 v62, s4, 42
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s5, 43
-; SI-NEXT: v_readfirstlane_b32 s5, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v7, v29
-; SI-NEXT: v_mov_b32_e32 v29, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s4, v34
+; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v41
; SI-NEXT: s_or_b32 s18, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v61
-; SI-NEXT: s_or_b32 s19, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v24
-; SI-NEXT: s_or_b32 s16, s5, s4
-; SI-NEXT: v_mov_b32_e32 v1, v53
-; SI-NEXT: v_mov_b32_e32 v34, v61
-; SI-NEXT: v_mov_b32_e32 v21, v24
+; SI-NEXT: v_readfirstlane_b32 s5, v34
+; SI-NEXT: v_mov_b32_e32 v3, v41
+; SI-NEXT: v_mov_b32_e32 v41, v34
; SI-NEXT: s_lshr_b32 s64, s25, 8
; SI-NEXT: s_lshr_b32 s54, s23, 8
; SI-NEXT: s_lshr_b32 s52, s21, 8
-; SI-NEXT: s_lshr_b32 s50, s19, 8
; SI-NEXT: s_lshr_b64 s[62:63], s[20:21], 8
-; SI-NEXT: s_lshr_b64 s[96:97], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[98:99], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 8
-; SI-NEXT: v_mov_b32_e32 v13, v12
-; SI-NEXT: v_bfe_u32 v24, v12, 8, 8
-; SI-NEXT: v_mov_b32_e32 v12, v48
-; SI-NEXT: v_bfe_u32 v48, v48, 8, 8
-; SI-NEXT: v_bfe_u32 v61, v59, 8, 8
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_mov_b32_e32 v18, v40
+; SI-NEXT: v_bfe_u32 v34, v19, 8, 8
+; SI-NEXT: v_mov_b32_e32 v14, v16
+; SI-NEXT: v_mov_b32_e32 v16, v38
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s4, v1
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s19, s5, s4
; SI-NEXT: v_readfirstlane_b32 s4, v11
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_bfe_u32 v18, v11, 8, 8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s5, v16
+; SI-NEXT: v_readfirstlane_b32 s5, v9
+; SI-NEXT: s_or_b32 s16, s5, s4
+; SI-NEXT: v_readfirstlane_b32 s5, v47
+; SI-NEXT: v_mov_b32_e32 v27, v1
+; SI-NEXT: s_lshr_b32 s50, s19, 8
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s17, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v44
+; SI-NEXT: v_readfirstlane_b32 s4, v25
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v42
+; SI-NEXT: v_readfirstlane_b32 s5, v44
; SI-NEXT: s_or_b32 s14, s5, s4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s4, v6
+; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v46
+; SI-NEXT: v_readfirstlane_b32 s5, v10
; SI-NEXT: s_or_b32 s15, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v8
+; SI-NEXT: v_readfirstlane_b32 s4, v46
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v23
+; SI-NEXT: v_readfirstlane_b32 s5, v33
; SI-NEXT: s_or_b32 s12, s5, s4
; SI-NEXT: v_readfirstlane_b32 s4, v5
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v3
+; SI-NEXT: v_readfirstlane_b32 s5, v36
; SI-NEXT: s_or_b32 s13, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v27
+; SI-NEXT: v_readfirstlane_b32 s4, v31
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v28
+; SI-NEXT: v_readfirstlane_b32 s5, v32
; SI-NEXT: s_or_b32 s10, s5, s4
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v45
+; SI-NEXT: v_readfirstlane_b32 s5, v2
; SI-NEXT: s_or_b32 s11, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v36
+; SI-NEXT: v_readfirstlane_b32 s4, v45
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v57
+; SI-NEXT: v_readfirstlane_b32 s5, v56
; SI-NEXT: s_or_b32 s8, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v59
+; SI-NEXT: v_readfirstlane_b32 s4, v38
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v56
+; SI-NEXT: v_readfirstlane_b32 s5, v57
; SI-NEXT: s_or_b32 s9, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v26
+; SI-NEXT: v_readfirstlane_b32 s4, v61
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v25
+; SI-NEXT: v_readfirstlane_b32 s5, v58
; SI-NEXT: s_or_b32 s6, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v52
+; SI-NEXT: v_readfirstlane_b32 s4, v51
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v50
+; SI-NEXT: v_readfirstlane_b32 s5, v60
; SI-NEXT: s_or_b32 s7, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s4, v49
+; SI-NEXT: v_readfirstlane_b32 s4, v37
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v54
-; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_readfirstlane_b32 s5, v53
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_readfirstlane_b32 s5, v54
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s5, s46, s5
; SI-NEXT: s_lshr_b64 vcc, s[4:5], 24
@@ -188781,6 +188256,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 49
; SI-NEXT: s_lshr_b64 vcc, s[4:5], 8
; SI-NEXT: v_writelane_b32 v62, vcc_lo, 46
+; SI-NEXT: v_bfe_u32 v37, v1, 8, 8
+; SI-NEXT: v_mov_b32_e32 v1, v54
; SI-NEXT: s_lshr_b32 s48, s17, 8
; SI-NEXT: s_lshr_b32 s67, s15, 8
; SI-NEXT: s_lshr_b32 s65, s13, 8
@@ -188788,6 +188265,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s53, s9, 8
; SI-NEXT: s_lshr_b32 s51, s7, 8
; SI-NEXT: s_lshr_b32 s49, s5, 8
+; SI-NEXT: s_lshr_b64 s[96:97], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[98:99], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 8
; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[38:39], s[16:17], 8
@@ -188809,171 +188289,166 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 47
; SI-NEXT: s_mov_b64 vcc, 0
-; SI-NEXT: v_mov_b32_e32 v57, v30
-; SI-NEXT: v_bfe_u32 v50, v30, 8, 8
-; SI-NEXT: v_mov_b32_e32 v56, v47
-; SI-NEXT: v_mov_b32_e32 v46, v33
-; SI-NEXT: v_mov_b32_e32 v30, v32
-; SI-NEXT: v_mov_b32_e32 v32, v31
-; SI-NEXT: v_mov_b32_e32 v31, v10
-; SI-NEXT: v_mov_b32_e32 v54, v9
-; SI-NEXT: v_bfe_u32 v42, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v9, v40, 8, 8
+; SI-NEXT: v_mov_b32_e32 v57, v13
+; SI-NEXT: v_bfe_u32 v47, v13, 8, 8
+; SI-NEXT: v_mov_b32_e32 v13, v29
+; SI-NEXT: v_bfe_u32 v53, v29, 8, 8
+; SI-NEXT: v_mov_b32_e32 v29, v21
+; SI-NEXT: v_bfe_u32 v36, v17, 8, 8
+; SI-NEXT: v_bfe_u32 v21, v42, 8, 8
+; SI-NEXT: v_bfe_u32 v19, v7, 8, 8
+; SI-NEXT: v_mov_b32_e32 v40, v8
+; SI-NEXT: v_bfe_u32 v56, v8, 8, 8
+; SI-NEXT: v_bfe_u32 v2, v5, 8, 8
+; SI-NEXT: v_mov_b32_e32 v54, v4
; SI-NEXT: v_bfe_u32 v45, v4, 8, 8
-; SI-NEXT: v_bfe_u32 v47, v52, 8, 8
-; SI-NEXT: v_bfe_u32 v33, v1, 8, 8
-; SI-NEXT: v_mov_b32_e32 v3, v14
-; SI-NEXT: v_mov_b32_e32 v25, v59
-; SI-NEXT: v_mov_b32_e32 v1, v52
-; SI-NEXT: v_mov_b32_e32 v44, v11
+; SI-NEXT: v_bfe_u32 v44, v38, 8, 8
+; SI-NEXT: v_bfe_u32 v58, v51, 8, 8
+; SI-NEXT: v_mov_b32_e32 v33, v1
+; SI-NEXT: v_bfe_u32 v46, v1, 8, 8
+; SI-NEXT: v_mov_b32_e32 v1, v51
+; SI-NEXT: v_mov_b32_e32 v10, v17
; SI-NEXT: s_branch .LBB95_3
; SI-NEXT: .LBB95_2:
-; SI-NEXT: v_mov_b32_e32 v60, v16
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v7, v29
-; SI-NEXT: v_mov_b32_e32 v29, v6
-; SI-NEXT: v_mov_b32_e32 v39, v5
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v12, v26
+; SI-NEXT: v_mov_b32_e32 v26, v7
+; SI-NEXT: v_mov_b32_e32 v20, v43
+; SI-NEXT: v_mov_b32_e32 v43, v42
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v18, v40
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: v_mov_b32_e32 v51, v42
+; SI-NEXT: v_mov_b32_e32 v35, v37
; SI-NEXT: v_writelane_b32 v62, s4, 0
; SI-NEXT: v_writelane_b32 v62, s5, 1
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v41, v21
+; SI-NEXT: v_mov_b32_e32 v52, v45
; SI-NEXT: v_writelane_b32 v62, s4, 2
; SI-NEXT: v_writelane_b32 v62, s5, 3
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v21, v24
+; SI-NEXT: v_mov_b32_e32 v50, v2
; SI-NEXT: v_writelane_b32 v62, s4, 4
; SI-NEXT: v_writelane_b32 v62, s5, 5
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v40, v34
+; SI-NEXT: v_mov_b32_e32 v49, v11
; SI-NEXT: v_writelane_b32 v62, s4, 6
; SI-NEXT: v_writelane_b32 v62, s5, 7
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v34, v61
+; SI-NEXT: v_mov_b32_e32 v48, v9
; SI-NEXT: v_writelane_b32 v62, s4, 8
; SI-NEXT: v_writelane_b32 v62, s5, 9
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v13, v12
+; SI-NEXT: v_mov_b32_e32 v39, v47
; SI-NEXT: v_writelane_b32 v62, s4, 10
; SI-NEXT: v_writelane_b32 v62, s5, 11
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v12, v48
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v3, v41
; SI-NEXT: v_writelane_b32 v62, s4, 12
; SI-NEXT: v_writelane_b32 v62, s5, 13
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v57, v30
+; SI-NEXT: v_mov_b32_e32 v41, v34
; SI-NEXT: v_writelane_b32 v62, s4, 14
; SI-NEXT: v_writelane_b32 v62, s5, 15
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v58, v11
+; SI-NEXT: v_mov_b32_e32 v57, v13
; SI-NEXT: v_writelane_b32 v62, s4, 16
; SI-NEXT: v_writelane_b32 v62, s5, 17
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v56, v47
+; SI-NEXT: v_mov_b32_e32 v13, v29
; SI-NEXT: v_writelane_b32 v62, s4, 18
; SI-NEXT: v_writelane_b32 v62, s5, 19
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v46, v33
+; SI-NEXT: v_mov_b32_e32 v33, v54
; SI-NEXT: v_writelane_b32 v62, s4, 20
; SI-NEXT: v_writelane_b32 v62, s5, 21
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v30, v32
+; SI-NEXT: v_mov_b32_e32 v29, v6
; SI-NEXT: v_writelane_b32 v62, s4, 22
; SI-NEXT: v_writelane_b32 v62, s5, 23
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v32, v31
+; SI-NEXT: v_mov_b32_e32 v28, v8
; SI-NEXT: v_writelane_b32 v62, s4, 24
; SI-NEXT: v_writelane_b32 v62, s5, 25
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v31, v10
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v55, v5
; SI-NEXT: v_writelane_b32 v62, s4, 26
; SI-NEXT: v_writelane_b32 v62, s5, 27
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v54, v9
+; SI-NEXT: v_mov_b32_e32 v54, v4
; SI-NEXT: v_writelane_b32 v62, s4, 28
; SI-NEXT: v_writelane_b32 v62, s5, 29
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v55, v4
+; SI-NEXT: v_mov_b32_e32 v22, v1
; SI-NEXT: v_writelane_b32 v62, s4, 30
; SI-NEXT: v_writelane_b32 v62, s5, 31
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v22, v2
+; SI-NEXT: s_mov_b64 vcc, -1
; SI-NEXT: v_writelane_b32 v62, s4, 32
; SI-NEXT: v_writelane_b32 v62, s5, 33
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v2, v1
+; SI-NEXT: v_mov_b32_e32 v1, v51
; SI-NEXT: v_writelane_b32 v62, s4, 34
; SI-NEXT: v_writelane_b32 v62, s5, 35
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v17, v43
+; SI-NEXT: v_mov_b32_e32 v16, v38
; SI-NEXT: v_writelane_b32 v62, s4, 36
; SI-NEXT: v_writelane_b32 v62, s5, 37
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_mov_b64 vcc, -1
-; SI-NEXT: v_writelane_b32 v62, s4, 38
-; SI-NEXT: v_writelane_b32 v62, s5, 39
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v25, v59
-; SI-NEXT: v_writelane_b32 v62, s4, 40
-; SI-NEXT: v_writelane_b32 v62, s5, 41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v1, v52
-; SI-NEXT: v_writelane_b32 v62, s4, 42
-; SI-NEXT: v_writelane_b32 v62, s5, 43
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $sgpr28
; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $sgpr26
; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $sgpr24
; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $sgpr22
; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $sgpr20
; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $sgpr18
; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr96
; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $sgpr55
; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr46
; SI-NEXT: ; implicit-def: $sgpr16
; SI-NEXT: ; implicit-def: $sgpr38
; SI-NEXT: ; implicit-def: $sgpr36
@@ -188998,6 +188473,15 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: ; implicit-def: $sgpr86
; SI-NEXT: ; implicit-def: $sgpr84
; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: v_writelane_b32 v62, s4, 38
+; SI-NEXT: v_writelane_b32 v62, s5, 39
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 40
+; SI-NEXT: v_writelane_b32 v62, s5, 41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 42
+; SI-NEXT: v_writelane_b32 v62, s5, 43
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: v_writelane_b32 v62, s4, 44
; SI-NEXT: v_writelane_b32 v62, s5, 45
; SI-NEXT: v_writelane_b32 v62, s80, 46
@@ -189008,143 +188492,146 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v62, s81, 49
; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: .LBB95_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v14, v17
-; SI-NEXT: v_mov_b32_e32 v17, v39
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v11, v13
+; SI-NEXT: v_mov_b32_e32 v13, v18
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v51, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, vcc
-; SI-NEXT: v_mov_b32_e32 v23, v2
-; SI-NEXT: v_mov_b32_e32 v59, v34
-; SI-NEXT: v_mov_b32_e32 v2, v25
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_mov_b32_e32 v8, v27
+; SI-NEXT: v_mov_b32_e32 v31, v28
+; SI-NEXT: v_mov_b32_e32 v32, v29
; SI-NEXT: s_cbranch_vccnz .LBB95_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_mov_b32_e32 v52, v29
-; SI-NEXT: v_mov_b32_e32 v29, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v18, v14
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mov_b32_e32 v15, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v53
+; SI-NEXT: v_mov_b32_e32 v29, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v17, v15
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v35
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v55
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v53, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v16
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v54
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_readfirstlane_b32 s4, v15
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_cvt_f16_f32_e32 v16, v2
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v55, v4
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v54
-; SI-NEXT: v_cvt_f32_f16_e32 v9, v31
+; SI-NEXT: v_cvt_f16_f32_e32 v54, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v55
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v40
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v55, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_cvt_f16_f32_e32 v40, v6
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v54, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v42
+; SI-NEXT: v_mov_b32_e32 v28, v26
+; SI-NEXT: v_mov_b32_e32 v26, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v31, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v10, v32
-; SI-NEXT: v_cvt_f32_f16_e32 v11, v57
-; SI-NEXT: v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v42, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v11
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v10
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v57
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
-; SI-NEXT: v_cvt_f16_f32_e32 v57, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v13
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v57, v12
+; SI-NEXT: v_bfe_u32 v53, v11, 8, 8
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT: v_bfe_u32 v50, v57, 8, 8
-; SI-NEXT: v_bfe_u32 v48, v12, 8, 8
-; SI-NEXT: v_bfe_u32 v43, v32, 8, 8
-; SI-NEXT: v_bfe_u32 v24, v13, 8, 8
-; SI-NEXT: v_bfe_u32 v20, v31, 8, 8
-; SI-NEXT: v_bfe_u32 v19, v54, 8, 8
-; SI-NEXT: v_bfe_u32 v42, v6, 8, 8
-; SI-NEXT: v_bfe_u32 v45, v55, 8, 8
-; SI-NEXT: v_bfe_u32 v61, v2, 8, 8
-; SI-NEXT: v_bfe_u32 v33, v53, 8, 8
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: v_bfe_u32 v47, v57, 8, 8
+; SI-NEXT: v_bfe_u32 v36, v10, 8, 8
+; SI-NEXT: v_bfe_u32 v21, v42, 8, 8
+; SI-NEXT: v_bfe_u32 v9, v13, 8, 8
+; SI-NEXT: v_bfe_u32 v37, v8, 8, 8
+; SI-NEXT: v_bfe_u32 v19, v7, 8, 8
+; SI-NEXT: v_bfe_u32 v56, v40, 8, 8
+; SI-NEXT: v_bfe_u32 v45, v54, 8, 8
+; SI-NEXT: v_bfe_u32 v44, v16, 8, 8
+; SI-NEXT: v_bfe_u32 v58, v1, 8, 8
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v44
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT: v_cvt_f16_f32_e32 v44, v7
-; SI-NEXT: v_readfirstlane_b32 s5, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v33
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v33, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_bfe_u32 v46, v33, 8, 8
+; SI-NEXT: v_readfirstlane_b32 s5, v15
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s5, v53
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: v_readfirstlane_b32 s5, v33
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_lshr_b64 vcc, s[4:5], 24
; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
; SI-NEXT: s_lshr_b32 s49, s5, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_readfirstlane_b32 s7, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_readfirstlane_b32 s7, v15
+; SI-NEXT: v_mov_b32_e32 v15, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v60
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readfirstlane_b32 s7, v3
+; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s8, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v52
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_lshr_b64 s[82:83], s[6:7], 24
; SI-NEXT: s_lshr_b64 s[84:85], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[86:87], s[6:7], 8
-; SI-NEXT: s_lshr_b32 s51, s7, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: s_lshr_b64 s[86:87], s[6:7], 8
+; SI-NEXT: s_lshr_b32 s51, s7, 8
; SI-NEXT: v_readfirstlane_b32 s8, v14
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s9, v14
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readfirstlane_b32 s9, v2
+; SI-NEXT: v_readfirstlane_b32 s9, v16
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s10, v14
@@ -189153,24 +188640,22 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[60:61], s[8:9], 8
; SI-NEXT: s_lshr_b32 s53, s9, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s10, v14
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s11, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v50
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readfirstlane_b32 s11, v55
+; SI-NEXT: v_readfirstlane_b32 s11, v54
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s12, v14
@@ -189180,24 +188665,24 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[76:77], s[10:11], 8
; SI-NEXT: s_lshr_b32 s55, s11, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s13, v14
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: v_readfirstlane_b32 s13, v5
+; SI-NEXT: v_readfirstlane_b32 s13, v55
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s14, v14
@@ -189207,26 +188692,28 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[12:13], 8
; SI-NEXT: s_lshr_b32 s65, s13, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s14, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v51
; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s15, v14
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: v_readfirstlane_b32 s15, v6
+; SI-NEXT: v_readfirstlane_b32 s15, v40
; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v41
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v49
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
; SI-NEXT: s_lshr_b64 s[94:95], s[14:15], 16
@@ -189235,19 +188722,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[30:31], s[14:15], 8
; SI-NEXT: s_lshr_b32 s67, s15, 8
; SI-NEXT: v_readfirstlane_b32 s16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v21
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v48
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s17, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v16
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v39
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readfirstlane_b32 s17, v44
+; SI-NEXT: v_readfirstlane_b32 s17, v7
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s18, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v40
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v38
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
@@ -189256,139 +188743,132 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[38:39], s[16:17], 8
; SI-NEXT: s_lshr_b32 s48, s17, 8
; SI-NEXT: v_readfirstlane_b32 s18, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v60
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v29
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s19, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v59
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v41
; SI-NEXT: s_or_b32 s18, s19, s18
-; SI-NEXT: v_readfirstlane_b32 s19, v54
+; SI-NEXT: v_readfirstlane_b32 s19, v8
; SI-NEXT: s_lshl_b32 s19, s19, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s20, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v18
; SI-NEXT: s_or_b32 s19, s20, s19
; SI-NEXT: s_lshr_b64 s[96:97], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[98:99], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 8
-; SI-NEXT: s_lshr_b32 s50, s19, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 8
+; SI-NEXT: s_lshr_b32 s50, s19, 8
; SI-NEXT: v_readfirstlane_b32 s20, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v61
; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s21, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v58
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v59
; SI-NEXT: s_or_b32 s20, s21, s20
-; SI-NEXT: v_readfirstlane_b32 s21, v31
+; SI-NEXT: v_readfirstlane_b32 s21, v42
; SI-NEXT: s_lshl_b32 s21, s21, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s22, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v39
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v22
; SI-NEXT: s_or_b32 s21, s22, s21
; SI-NEXT: s_lshr_b64 s[62:63], s[20:21], 8
; SI-NEXT: s_lshr_b32 s52, s21, 8
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s22, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v29
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v26
; SI-NEXT: s_lshl_b32 s22, s22, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s23, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v52
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v28
; SI-NEXT: s_or_b32 s22, s23, s22
-; SI-NEXT: v_readfirstlane_b32 s23, v32
+; SI-NEXT: v_readfirstlane_b32 s23, v10
; SI-NEXT: s_lshl_b32 s23, s23, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s24, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v17
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v2
; SI-NEXT: s_or_b32 s23, s24, s23
; SI-NEXT: s_lshr_b32 s54, s23, 8
+; SI-NEXT: v_bfe_u32 v2, v55, 8, 8
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s24, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v17
; SI-NEXT: s_lshl_b32 s24, s24, 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s25, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v30
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v31
; SI-NEXT: s_or_b32 s24, s25, s24
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v14
-; SI-NEXT: v_readfirstlane_b32 s25, v30
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: v_bfe_u32 v35, v30, 8, 8
+; SI-NEXT: v_cvt_f16_f32_e32 v31, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s25, v31
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: s_lshl_b32 s25, s25, 16
+; SI-NEXT: v_bfe_u32 v23, v31, 8, 8
; SI-NEXT: v_readfirstlane_b32 s26, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v22
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v51
; SI-NEXT: s_or_b32 s25, s26, s25
; SI-NEXT: s_lshr_b32 s64, s25, 8
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s26, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v23
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v20
; SI-NEXT: s_lshl_b32 s26, s26, 16
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s27, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v46
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v32
; SI-NEXT: s_or_b32 s26, s27, s26
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v46, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v18
-; SI-NEXT: v_bfe_u32 v18, v44, 8, 8
-; SI-NEXT: v_readfirstlane_b32 s27, v46
+; SI-NEXT: v_cvt_f16_f32_e32 v32, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v43
+; SI-NEXT: v_readfirstlane_b32 s27, v32
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: s_lshl_b32 s27, s27, 16
-; SI-NEXT: v_bfe_u32 v37, v46, 8, 8
+; SI-NEXT: v_bfe_u32 v24, v32, 8, 8
; SI-NEXT: v_readfirstlane_b32 s28, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s27, s28, s27
; SI-NEXT: s_lshr_b32 s66, s27, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s28, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s28, s28, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s29, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v56
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v30
; SI-NEXT: s_or_b32 s28, s29, s28
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v56, v14
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v14
; SI-NEXT: v_cvt_f32_f16_e32 v14, v15
-; SI-NEXT: v_bfe_u32 v15, v5, 8, 8
-; SI-NEXT: v_readfirstlane_b32 s29, v56
+; SI-NEXT: v_readfirstlane_b32 s29, v30
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: s_lshl_b32 s29, s29, 16
-; SI-NEXT: v_bfe_u32 v38, v56, 8, 8
+; SI-NEXT: v_bfe_u32 v34, v30, 8, 8
; SI-NEXT: v_readfirstlane_b32 s40, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s29, s40, s29
; SI-NEXT: s_lshr_b32 s68, s29, 8
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -189396,62 +188876,60 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s40, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s40, s40, 16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s41, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s40, s41, s40
-; SI-NEXT: v_readfirstlane_b32 s41, v57
+; SI-NEXT: v_readfirstlane_b32 s41, v11
+; SI-NEXT: s_lshl_b32 s41, s41, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: s_lshl_b32 s41, s41, 16
; SI-NEXT: v_readfirstlane_b32 s42, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s41, s42, s41
; SI-NEXT: s_lshr_b32 s69, s41, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s42, v14
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s43, v14
; SI-NEXT: s_or_b32 s42, s43, s42
-; SI-NEXT: v_readfirstlane_b32 s43, v12
+; SI-NEXT: v_readfirstlane_b32 s43, v57
; SI-NEXT: s_lshl_b32 s43, s43, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v3
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s44, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s43, s44, s43
; SI-NEXT: s_lshr_b32 s70, s43, 8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s44, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s44, s44, 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v14, v1
-; SI-NEXT: v_mov_b32_e32 v1, v3
-; SI-NEXT: v_bfe_u32 v47, v1, 8, 8
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_readfirstlane_b32 s45, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: s_or_b32 s44, s45, s44
; SI-NEXT: v_readfirstlane_b32 s45, v13
; SI-NEXT: s_lshl_b32 s45, s45, 16
@@ -189538,203 +189016,188 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v62, vcc_hi, 47
; SI-NEXT: s_lshr_b32 s71, s45, 8
; SI-NEXT: .LBB95_5: ; %end
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 0
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 1
-; SI-NEXT: s_lshl_b32 s47, vcc_lo, 8
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 2
+; SI-NEXT: v_readlane_b32 s47, v62, 0
+; SI-NEXT: s_lshl_b32 s47, s47, 8
; SI-NEXT: s_and_b32 s44, s44, 0xff
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 3
; SI-NEXT: s_or_b32 s44, s44, s47
-; SI-NEXT: s_and_b32 s47, vcc_lo, 0xff
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 4
-; SI-NEXT: s_lshl_b32 s57, vcc_lo, 24
+; SI-NEXT: v_readlane_b32 s47, v62, 2
+; SI-NEXT: s_and_b32 s47, s47, 0xff
+; SI-NEXT: v_readlane_b32 s57, v62, 4
+; SI-NEXT: s_lshl_b32 s57, s57, 24
; SI-NEXT: s_lshl_b32 s47, s47, 16
; SI-NEXT: s_or_b32 s47, s57, s47
; SI-NEXT: s_and_b32 s44, s44, 0xffff
; SI-NEXT: s_or_b32 s44, s44, s47
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v16, s44
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_mov_b32_e32 v14, s44
; SI-NEXT: s_and_b32 s44, s45, 0xff
; SI-NEXT: s_lshl_b32 s45, s71, 8
; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
; SI-NEXT: s_or_b32 s44, s44, s45
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v24
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 24, v9
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: v_or_b32_e32 v13, v21, v13
+; SI-NEXT: v_or_b32_e32 v13, v18, v13
; SI-NEXT: v_or_b32_e32 v13, s44, v13
; SI-NEXT: v_readlane_b32 s44, v62, 6
-; SI-NEXT: v_readlane_b32 s45, v62, 7
; SI-NEXT: s_lshl_b32 s44, s44, 8
; SI-NEXT: s_and_b32 s42, s42, 0xff
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 5
; SI-NEXT: s_or_b32 s42, s42, s44
; SI-NEXT: v_readlane_b32 s44, v62, 8
-; SI-NEXT: v_readlane_b32 s45, v62, 9
; SI-NEXT: s_and_b32 s44, s44, 0xff
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 10
-; SI-NEXT: s_lshl_b32 s45, vcc_lo, 24
+; SI-NEXT: v_readlane_b32 s45, v62, 10
+; SI-NEXT: s_lshl_b32 s45, s45, 24
; SI-NEXT: s_lshl_b32 s44, s44, 16
; SI-NEXT: s_or_b32 s44, s45, s44
; SI-NEXT: s_and_b32 s42, s42, 0xffff
; SI-NEXT: s_or_b32 s42, s42, s44
-; SI-NEXT: v_mov_b32_e32 v21, s42
+; SI-NEXT: v_mov_b32_e32 v18, s42
; SI-NEXT: s_and_b32 s42, s43, 0xff
; SI-NEXT: s_lshl_b32 s43, s70, 8
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v57
; SI-NEXT: s_or_b32 s42, s42, s43
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v23, 24, v48
+; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v47
; SI-NEXT: s_and_b32 s42, s42, 0xffff
-; SI-NEXT: v_or_b32_e32 v12, v23, v12
+; SI-NEXT: v_or_b32_e32 v12, v25, v12
; SI-NEXT: v_or_b32_e32 v12, s42, v12
; SI-NEXT: v_readlane_b32 s42, v62, 12
-; SI-NEXT: v_readlane_b32 s43, v62, 13
; SI-NEXT: s_lshl_b32 s42, s42, 8
; SI-NEXT: s_and_b32 s40, s40, 0xff
; SI-NEXT: s_or_b32 s40, s40, s42
; SI-NEXT: v_readlane_b32 s42, v62, 14
-; SI-NEXT: v_readlane_b32 s43, v62, 15
; SI-NEXT: s_and_b32 s42, s42, 0xff
-; SI-NEXT: v_readlane_b32 s44, v62, 16
-; SI-NEXT: s_lshl_b32 s43, s44, 24
+; SI-NEXT: v_readlane_b32 s43, v62, 16
+; SI-NEXT: s_lshl_b32 s43, s43, 24
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_or_b32 s42, s43, s42
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_or_b32 s40, s40, s42
-; SI-NEXT: v_mov_b32_e32 v23, s40
+; SI-NEXT: v_mov_b32_e32 v25, s40
; SI-NEXT: s_and_b32 s40, s41, 0xff
; SI-NEXT: s_lshl_b32 s41, s69, 8
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v57
+; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
; SI-NEXT: s_or_b32 s40, s40, s41
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 24, v50
+; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v53
; SI-NEXT: s_and_b32 s40, s40, 0xffff
-; SI-NEXT: v_or_b32_e32 v11, v24, v11
+; SI-NEXT: v_or_b32_e32 v11, v26, v11
; SI-NEXT: v_or_b32_e32 v11, s40, v11
; SI-NEXT: v_readlane_b32 s40, v62, 18
-; SI-NEXT: v_readlane_b32 s41, v62, 19
; SI-NEXT: s_lshl_b32 s40, s40, 8
; SI-NEXT: s_and_b32 s28, s28, 0xff
; SI-NEXT: s_or_b32 s28, s28, s40
; SI-NEXT: v_readlane_b32 s40, v62, 20
-; SI-NEXT: v_readlane_b32 s41, v62, 21
; SI-NEXT: s_and_b32 s40, s40, 0xff
-; SI-NEXT: v_readlane_b32 s42, v62, 22
-; SI-NEXT: s_lshl_b32 s41, s42, 24
+; SI-NEXT: v_readlane_b32 s41, v62, 22
+; SI-NEXT: s_lshl_b32 s41, s41, 24
; SI-NEXT: s_lshl_b32 s40, s40, 16
; SI-NEXT: s_or_b32 s40, s41, s40
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_or_b32 s28, s28, s40
-; SI-NEXT: v_mov_b32_e32 v24, s28
+; SI-NEXT: v_mov_b32_e32 v26, s28
; SI-NEXT: s_and_b32 s28, s29, 0xff
; SI-NEXT: s_lshl_b32 s29, s68, 8
-; SI-NEXT: v_and_b32_e32 v25, 0xff, v56
+; SI-NEXT: v_and_b32_e32 v27, 0xff, v30
; SI-NEXT: s_or_b32 s28, s28, s29
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v38
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v28, 24, v34
; SI-NEXT: s_and_b32 s28, s28, 0xffff
-; SI-NEXT: v_or_b32_e32 v25, v26, v25
-; SI-NEXT: v_or_b32_e32 v25, s28, v25
+; SI-NEXT: v_or_b32_e32 v27, v28, v27
+; SI-NEXT: v_or_b32_e32 v27, s28, v27
; SI-NEXT: v_readlane_b32 s28, v62, 24
-; SI-NEXT: v_readlane_b32 s29, v62, 25
; SI-NEXT: s_lshl_b32 s28, s28, 8
; SI-NEXT: s_and_b32 s26, s26, 0xff
; SI-NEXT: s_or_b32 s26, s26, s28
; SI-NEXT: v_readlane_b32 s28, v62, 26
-; SI-NEXT: v_readlane_b32 s29, v62, 27
; SI-NEXT: s_and_b32 s28, s28, 0xff
-; SI-NEXT: v_readlane_b32 s40, v62, 28
-; SI-NEXT: s_lshl_b32 s29, s40, 24
+; SI-NEXT: v_readlane_b32 s29, v62, 28
+; SI-NEXT: s_lshl_b32 s29, s29, 24
; SI-NEXT: s_lshl_b32 s28, s28, 16
; SI-NEXT: s_or_b32 s28, s29, s28
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_or_b32 s26, s26, s28
-; SI-NEXT: v_mov_b32_e32 v26, s26
+; SI-NEXT: v_mov_b32_e32 v28, s26
; SI-NEXT: s_and_b32 s26, s27, 0xff
; SI-NEXT: s_lshl_b32 s27, s66, 8
-; SI-NEXT: v_and_b32_e32 v27, 0xff, v46
+; SI-NEXT: v_and_b32_e32 v29, 0xff, v32
; SI-NEXT: s_or_b32 s26, s26, s27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v28, 24, v37
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v24, 24, v24
; SI-NEXT: s_and_b32 s26, s26, 0xffff
-; SI-NEXT: v_or_b32_e32 v27, v28, v27
-; SI-NEXT: v_or_b32_e32 v27, s26, v27
+; SI-NEXT: v_or_b32_e32 v24, v24, v29
+; SI-NEXT: v_or_b32_e32 v24, s26, v24
; SI-NEXT: v_readlane_b32 s26, v62, 30
-; SI-NEXT: v_readlane_b32 s27, v62, 31
; SI-NEXT: s_lshl_b32 s26, s26, 8
; SI-NEXT: s_and_b32 s24, s24, 0xff
; SI-NEXT: s_or_b32 s24, s24, s26
; SI-NEXT: v_readlane_b32 s26, v62, 32
-; SI-NEXT: v_readlane_b32 s27, v62, 33
; SI-NEXT: s_and_b32 s26, s26, 0xff
-; SI-NEXT: v_readlane_b32 s28, v62, 34
-; SI-NEXT: s_lshl_b32 s27, s28, 24
+; SI-NEXT: v_readlane_b32 s27, v62, 34
+; SI-NEXT: s_lshl_b32 s27, s27, 24
; SI-NEXT: s_lshl_b32 s26, s26, 16
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 11
; SI-NEXT: s_or_b32 s26, s27, s26
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_or_b32 s24, s24, s26
-; SI-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v16, vcc, 4, v0
-; SI-NEXT: v_mov_b32_e32 v28, s24
+; SI-NEXT: v_add_i32_e32 v14, vcc, 4, v0
+; SI-NEXT: v_mov_b32_e32 v29, s24
; SI-NEXT: s_and_b32 s24, s25, 0xff
; SI-NEXT: s_lshl_b32 s25, s64, 8
-; SI-NEXT: v_and_b32_e32 v29, 0xff, v30
-; SI-NEXT: buffer_store_dword v13, v16, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v30, 0xff, v31
+; SI-NEXT: buffer_store_dword v13, v14, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v13, vcc, 8, v0
; SI-NEXT: s_or_b32 s24, s24, s25
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v30, 24, v35
-; SI-NEXT: buffer_store_dword v21, v13, s[0:3], 0 offen
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v23, 24, v23
+; SI-NEXT: buffer_store_dword v18, v13, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v13, vcc, 12, v0
; SI-NEXT: s_and_b32 s24, s24, 0xffff
-; SI-NEXT: v_or_b32_e32 v29, v30, v29
+; SI-NEXT: v_or_b32_e32 v23, v23, v30
; SI-NEXT: buffer_store_dword v12, v13, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v12, vcc, 16, v0
-; SI-NEXT: v_or_b32_e32 v29, s24, v29
-; SI-NEXT: buffer_store_dword v23, v12, s[0:3], 0 offen
+; SI-NEXT: v_or_b32_e32 v23, s24, v23
+; SI-NEXT: buffer_store_dword v25, v12, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v12, vcc, 20, v0
; SI-NEXT: v_readlane_b32 s24, v62, 36
; SI-NEXT: buffer_store_dword v11, v12, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v11, vcc, 24, v0
-; SI-NEXT: v_readlane_b32 s25, v62, 37
; SI-NEXT: s_lshl_b32 s24, s24, 8
; SI-NEXT: s_and_b32 s22, s22, 0xff
-; SI-NEXT: buffer_store_dword v24, v11, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v26, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 28, v0
; SI-NEXT: s_or_b32 s22, s22, s24
; SI-NEXT: v_readlane_b32 s24, v62, 38
-; SI-NEXT: buffer_store_dword v25, v11, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v27, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 32, v0
-; SI-NEXT: v_readlane_b32 s25, v62, 39
; SI-NEXT: s_and_b32 s24, s24, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 40
-; SI-NEXT: buffer_store_dword v26, v11, s[0:3], 0 offen
+; SI-NEXT: v_readlane_b32 s25, v62, 40
+; SI-NEXT: buffer_store_dword v28, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 36, v0
; SI-NEXT: s_lshl_b32 s24, s24, 16
-; SI-NEXT: s_lshl_b32 s25, s26, 24
-; SI-NEXT: buffer_store_dword v27, v11, s[0:3], 0 offen
+; SI-NEXT: s_lshl_b32 s25, s25, 24
+; SI-NEXT: buffer_store_dword v24, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 40, v0
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s24, s25, s24
-; SI-NEXT: buffer_store_dword v28, v11, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v29, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 44, v0
; SI-NEXT: s_or_b32 s22, s22, s24
-; SI-NEXT: buffer_store_dword v29, v11, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v23, v11, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v11, vcc, 48, v0
; SI-NEXT: v_mov_b32_e32 v12, s22
; SI-NEXT: s_and_b32 s22, s23, 0xff
; SI-NEXT: s_lshl_b32 s23, s54, 8
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v32
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
; SI-NEXT: buffer_store_dword v12, v11, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v43
+; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v36
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_or_b32_e32 v10, s22, v10
@@ -189742,11 +189205,10 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s22, s62, 8
; SI-NEXT: s_or_b32 s20, s20, s22
; SI-NEXT: v_readlane_b32 s22, v62, 42
-; SI-NEXT: v_readlane_b32 s23, v62, 43
; SI-NEXT: s_and_b32 s22, s22, 0xff
-; SI-NEXT: v_readlane_b32 s24, v62, 44
+; SI-NEXT: v_readlane_b32 s23, v62, 44
; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_lshl_b32 s23, s24, 24
+; SI-NEXT: s_lshl_b32 s23, s23, 24
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s22, s23, s22
; SI-NEXT: v_add_i32_e32 v11, vcc, 52, v0
@@ -189757,11 +189219,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v11, s20
; SI-NEXT: s_and_b32 s20, s21, 0xff
; SI-NEXT: s_lshl_b32 s21, s52, 8
-; SI-NEXT: v_and_b32_e32 v9, 0xff, v31
+; SI-NEXT: v_and_b32_e32 v9, 0xff, v42
; SI-NEXT: buffer_store_dword v11, v10, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v20
+; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v21
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_or_b32_e32 v9, s20, v9
@@ -189781,11 +189243,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s18
; SI-NEXT: s_and_b32 s18, s19, 0xff
; SI-NEXT: s_lshl_b32 s19, s50, 8
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v54
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
; SI-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v19
+; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v37
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_or_b32_e32 v8, s18, v8
@@ -189805,11 +189267,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v9, s16
; SI-NEXT: s_and_b32 s16, s17, 0xff
; SI-NEXT: s_lshl_b32 s17, s48, 8
-; SI-NEXT: v_and_b32_e32 v7, 0xff, v44
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
; SI-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v18
+; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v19
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_or_b32_e32 v7, s16, v7
@@ -189829,11 +189291,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s14
; SI-NEXT: s_and_b32 s14, s15, 0xff
; SI-NEXT: s_lshl_b32 s15, s67, 8
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v40
; SI-NEXT: buffer_store_dword v8, v7, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s14, s14, s15
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v42
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v56
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_or_b32_e32 v6, s14, v6
@@ -189853,11 +189315,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v7, s12
; SI-NEXT: s_and_b32 s12, s13, 0xff
; SI-NEXT: s_lshl_b32 s13, s65, 8
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v55
; SI-NEXT: buffer_store_dword v7, v6, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v2
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_or_b32_e32 v5, s12, v5
@@ -189877,7 +189339,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: s_and_b32 s10, s11, 0xff
; SI-NEXT: s_lshl_b32 s11, s55, 8
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v55
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v54
; SI-NEXT: buffer_store_dword v6, v5, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s10, s10, s11
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
@@ -189901,11 +189363,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v5, s8
; SI-NEXT: s_and_b32 s8, s9, 0xff
; SI-NEXT: s_lshl_b32 s9, s53, 8
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v16
; SI-NEXT: buffer_store_dword v5, v4, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v61
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v44
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_or_b32_e32 v2, s8, v2
@@ -189929,19 +189391,18 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v47
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v58
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: v_or_b32_e32 v1, s6, v1
; SI-NEXT: v_readlane_b32 s6, v62, 46
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: v_readlane_b32 s7, v62, 47
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s80, 0xff
-; SI-NEXT: v_readlane_b32 s8, v62, 48
+; SI-NEXT: v_readlane_b32 s7, v62, 48
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x74, v0
@@ -189953,11 +189414,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_and_b32 s4, s5, 0xff
; SI-NEXT: s_lshl_b32 s5, s49, 8
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v53
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v33
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v33
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v46
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
@@ -189977,13 +189438,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s45, v62, 17
-; SI-NEXT: v_readlane_b32 s43, v62, 23
-; SI-NEXT: v_readlane_b32 s41, v62, 29
-; SI-NEXT: v_readlane_b32 s29, v62, 35
-; SI-NEXT: v_readlane_b32 s27, v62, 41
-; SI-NEXT: v_readlane_b32 s25, v62, 45
-; SI-NEXT: v_readlane_b32 s9, v62, 49
; SI-NEXT: v_readlane_b32 s99, v63, 35
; SI-NEXT: v_readlane_b32 s98, v63, 34
; SI-NEXT: v_readlane_b32 s97, v63, 33
@@ -190021,8 +189475,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -191167,16 +190621,15 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_add_u32_e32 v11, vcc, 44, v39
; VI-NEXT: buffer_store_dword v1, v11, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v11, 8, v2
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v11, 8, v4
+; VI-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v11, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v11, vcc, 48, v39
@@ -191192,15 +190645,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v11, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v2
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v9, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -191283,16 +190735,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v4
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v0
; VI-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x58, v39
@@ -191338,16 +190788,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v0
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x68, v39
@@ -191367,16 +190815,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v0
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x70, v39
@@ -191396,16 +190842,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v0
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x78, v39
@@ -192365,20 +191809,20 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v55, v19
; GFX9-NEXT: v_mov_b32_e32 v19, s81
; GFX9-NEXT: .LBB95_5: ; %end
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
+; GFX9-NEXT: v_or_b32_sdwa v19, v21, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 8, v15
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v37, 8, v61
-; GFX9-NEXT: v_or_b32_sdwa v21, v21, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_or_b32_sdwa v35, v35, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v37, 8, v60
; GFX9-NEXT: v_or_b32_sdwa v36, v36, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v20
-; GFX9-NEXT: v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 8, v23
; GFX9-NEXT: v_or_b32_sdwa v23, v33, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 8, v59
@@ -192436,26 +191880,23 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 8, v21
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v22, v22, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v37
-; GFX9-NEXT: v_or_b32_sdwa v19, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v19, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 8, v22
+; GFX9-NEXT: v_or_b32_sdwa v22, v42, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v20, v20, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v43
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v18, v18, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:4
; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
; GFX9-NEXT: v_or_b32_sdwa v18, v40, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v16, v16, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -192468,9 +191909,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192487,9 +191927,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192504,11 +191943,11 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v33, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v18
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -192524,9 +191963,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192543,9 +191981,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192562,9 +191999,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192581,13 +192017,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:64
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
@@ -192596,13 +192031,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v15, v22, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192629,9 +192063,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: v_or_b32_sdwa v11, v11, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192654,9 +192087,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192679,9 +192111,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: v_or_b32_sdwa v7, v7, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192704,9 +192135,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192729,9 +192159,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -192748,8 +192177,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v52
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -193438,10 +192866,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v149, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v148, s0
; GFX11-NEXT: v_readlane_b32 s0, v78, 0
-; GFX11-NEXT: v_readlane_b32 s1, v78, 1
; GFX11-NEXT: v_mov_b32_e32 v82, s0
; GFX11-NEXT: .LBB95_5: ; %end
; GFX11-NEXT: v_lshlrev_b32_e32 v69, 8, v74
@@ -208414,16 +207841,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 16, v0
@@ -208443,16 +207868,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 24, v0
@@ -208472,16 +207895,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 32, v0
@@ -208501,16 +207922,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v0
@@ -208530,16 +207949,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 48, v0
@@ -208564,8 +207981,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v55, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -208655,8 +208071,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v51, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -208677,16 +208092,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x60, v0
@@ -208706,16 +208119,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x68, v0
@@ -208736,16 +208147,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x70, v0
@@ -208770,8 +208179,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v2, v50, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209550,9 +208958,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -209569,9 +208976,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -209613,12 +209019,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v21, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209639,12 +209045,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v23, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209665,12 +209071,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v25, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209691,12 +209097,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v27, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209717,12 +209123,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v29, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -209743,12 +209149,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -211829,12 +211235,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: .LBB99_3: ; %end
; SI-NEXT: s_lshl_b32 s17, s92, 8
; SI-NEXT: s_and_b32 s18, s60, 0xff
+; SI-NEXT: v_readlane_b32 s16, v41, 20
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v41, 20
-; SI-NEXT: v_readlane_b32 s19, v41, 21
-; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: v_readlane_b32 s20, v41, 22
-; SI-NEXT: s_lshl_b32 s19, s20, 24
+; SI-NEXT: s_and_b32 s18, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v41, 22
+; SI-NEXT: s_lshl_b32 s19, s16, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -211852,18 +211257,15 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: v_mov_b32_e32 v2, s17
; SI-NEXT: v_readlane_b32 s16, v41, 24
-; SI-NEXT: v_readlane_b32 s17, v41, 25
+; SI-NEXT: v_mov_b32_e32 v2, s17
; SI-NEXT: s_lshl_b32 s17, s16, 8
; SI-NEXT: s_and_b32 s18, s56, 0xff
-; SI-NEXT: v_readlane_b32 s21, v41, 23
+; SI-NEXT: v_readlane_b32 s16, v41, 26
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v41, 26
-; SI-NEXT: v_readlane_b32 s19, v41, 27
-; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: v_readlane_b32 s20, v41, 28
-; SI-NEXT: s_lshl_b32 s19, s20, 24
+; SI-NEXT: s_and_b32 s18, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v41, 28
+; SI-NEXT: s_lshl_b32 s19, s16, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -211999,13 +211401,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v14, s17
; SI-NEXT: s_lshl_b32 s17, s90, 8
; SI-NEXT: s_and_b32 s18, s46, 0xff
-; SI-NEXT: v_readlane_b32 s21, v41, 29
+; SI-NEXT: v_readlane_b32 s16, v41, 30
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v41, 30
-; SI-NEXT: v_readlane_b32 s19, v41, 31
-; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: v_readlane_b32 s20, v41, 32
-; SI-NEXT: s_lshl_b32 s19, s20, 24
+; SI-NEXT: s_and_b32 s18, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v41, 32
+; SI-NEXT: s_lshl_b32 s19, s16, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -212018,84 +211418,79 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s16, 0xff
; SI-NEXT: v_readlane_b32 s16, v43, 39
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s19, s16, 24
-; SI-NEXT: s_or_b32 s18, s19, s18
-; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 4, v0
-; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_lshl_b32 s19, s16, 24
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 8, v0
-; SI-NEXT: v_mov_b32_e32 v16, s17
-; SI-NEXT: v_readlane_b32 s16, v41, 34
+; SI-NEXT: s_or_b32 s18, s19, s18
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 12, v0
-; SI-NEXT: v_readlane_b32 s17, v41, 35
-; SI-NEXT: v_readlane_b32 s18, v41, 36
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: v_readlane_b32 s16, v41, 34
; SI-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, s17
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_and_b32 s17, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v41, 37
; SI-NEXT: buffer_store_dword v5, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 20, v0
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v41, 38
+; SI-NEXT: v_readlane_b32 s17, v41, 36
; SI-NEXT: buffer_store_dword v6, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
-; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v41, 38
; SI-NEXT: buffer_store_dword v7, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 28, v0
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v0
-; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 36, v0
-; SI-NEXT: v_readlane_b32 s17, v43, 44
+; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: buffer_store_dword v10, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 40, v0
+; SI-NEXT: v_readlane_b32 s17, v43, 44
+; SI-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen
+; SI-NEXT: v_add_i32_e32 v1, vcc, 44, v0
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s43, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
-; SI-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen
-; SI-NEXT: v_add_i32_e32 v1, vcc, 44, v0
-; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v43, 43
; SI-NEXT: buffer_store_dword v12, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 48, v0
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v43, 42
-; SI-NEXT: v_readlane_b32 s19, v41, 39
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: v_readlane_b32 s17, v43, 43
; SI-NEXT: buffer_store_dword v13, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 52, v0
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v43, 42
; SI-NEXT: buffer_store_dword v14, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 56, v0
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v41, 40
+; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: buffer_store_dword v15, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 60, v0
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v41, 41
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v41, 42
+; SI-NEXT: v_readlane_b32 s17, v41, 40
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v41, 43
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v41, 42
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v41, 44
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -212124,16 +211519,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: v_readlane_b32 s16, v41, 46
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v41, 47
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v41, 45
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v41, 48
-; SI-NEXT: v_readlane_b32 s17, v41, 49
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v41, 50
+; SI-NEXT: v_readlane_b32 s17, v41, 50
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x4c, v0
@@ -212159,15 +211551,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_readlane_b32 s14, v41, 52
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v41, 53
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v41, 54
-; SI-NEXT: v_readlane_b32 s15, v41, 55
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v41, 56
+; SI-NEXT: v_readlane_b32 s15, v41, 56
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x54, v0
@@ -212193,15 +211583,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: v_readlane_b32 s12, v41, 58
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v41, 59
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: v_readlane_b32 s12, v41, 60
-; SI-NEXT: v_readlane_b32 s13, v41, 61
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v41, 62
+; SI-NEXT: v_readlane_b32 s13, v41, 62
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x5c, v0
@@ -212227,15 +211615,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: v_readlane_b32 s10, v43, 0
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v43, 1
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
; SI-NEXT: v_readlane_b32 s10, v43, 2
-; SI-NEXT: v_readlane_b32 s11, v43, 3
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v43, 4
+; SI-NEXT: v_readlane_b32 s11, v43, 4
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x64, v0
@@ -212261,15 +211647,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_readlane_b32 s8, v43, 6
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v43, 7
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
; SI-NEXT: v_readlane_b32 s8, v43, 8
-; SI-NEXT: v_readlane_b32 s9, v43, 9
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v43, 10
+; SI-NEXT: v_readlane_b32 s9, v43, 10
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x6c, v0
@@ -212295,15 +211679,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_readlane_b32 s6, v43, 12
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s7, v43, 13
; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: v_readlane_b32 s6, v43, 14
-; SI-NEXT: v_readlane_b32 s7, v43, 15
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v43, 16
+; SI-NEXT: v_readlane_b32 s7, v43, 16
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x74, v0
@@ -212327,13 +211709,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s21, v41, 33
-; SI-NEXT: v_readlane_b32 s19, v41, 51
-; SI-NEXT: v_readlane_b32 s17, v41, 57
-; SI-NEXT: v_readlane_b32 s15, v41, 63
-; SI-NEXT: v_readlane_b32 s13, v43, 5
-; SI-NEXT: v_readlane_b32 s11, v43, 11
-; SI-NEXT: v_readlane_b32 s9, v43, 17
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s99, v40, 35
; SI-NEXT: v_readlane_b32 s98, v40, 34
@@ -214721,20 +214096,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v55, v19
; GFX9-NEXT: v_mov_b32_e32 v19, s81
; GFX9-NEXT: .LBB99_5: ; %end
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
+; GFX9-NEXT: v_or_b32_sdwa v19, v21, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 8, v15
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v37, 8, v61
-; GFX9-NEXT: v_or_b32_sdwa v21, v21, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_or_b32_sdwa v35, v35, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v37, 8, v60
; GFX9-NEXT: v_or_b32_sdwa v36, v36, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v20
-; GFX9-NEXT: v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 8, v23
; GFX9-NEXT: v_or_b32_sdwa v23, v33, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 8, v59
@@ -214792,26 +214167,23 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 8, v21
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v22, v22, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v37
-; GFX9-NEXT: v_or_b32_sdwa v19, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen
-; GFX9-NEXT: v_lshlrev_b32_e32 v19, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v19, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 8, v22
+; GFX9-NEXT: v_or_b32_sdwa v22, v42, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v20, v20, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 8, v43
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v18, v18, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:4
; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 8, v18
; GFX9-NEXT: v_or_b32_sdwa v18, v40, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v16, v16, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -214824,9 +214196,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214843,9 +214214,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:20
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214860,11 +214230,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v33, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v18
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -214880,9 +214250,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214899,9 +214268,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214918,9 +214286,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214937,13 +214304,12 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:60
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:64
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
@@ -214952,13 +214318,12 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v15, v22, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:68
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -214985,9 +214350,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: v_or_b32_sdwa v11, v11, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 8, v13
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -215010,9 +214374,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: v_or_b32_sdwa v9, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 8, v11
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -215035,9 +214398,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: v_or_b32_sdwa v7, v7, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -215060,9 +214422,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -215085,9 +214446,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -215104,8 +214464,7 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v52
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -215794,10 +215153,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v149, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v148, s0
; GFX11-NEXT: v_readlane_b32 s0, v78, 0
-; GFX11-NEXT: v_readlane_b32 s1, v78, 1
; GFX11-NEXT: v_mov_b32_e32 v82, s0
; GFX11-NEXT: .LBB99_5: ; %end
; GFX11-NEXT: v_lshlrev_b32_e32 v69, 8, v74
@@ -230364,21 +229722,21 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:76
; SI-NEXT: v_mul_f32_e32 v52, 1.0, v2
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v13
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v17
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v21
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v25
; SI-NEXT: v_mov_b32_e32 v50, v27
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v29
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
; SI-NEXT: v_mul_f32_e32 v54, 1.0, v4
@@ -230387,7 +229745,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e32 v48, 1.0, v7
; SI-NEXT: v_mul_f32_e32 v40, 1.0, v8
; SI-NEXT: v_mul_f32_e32 v61, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v10
+; SI-NEXT: v_mul_f32_e32 v56, 1.0, v10
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v11
; SI-NEXT: v_mul_f32_e32 v42, 1.0, v12
; SI-NEXT: v_mul_f32_e32 v46, 1.0, v14
@@ -230396,13 +229754,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e32 v63, 1.0, v18
; SI-NEXT: v_mul_f32_e32 v13, 1.0, v19
; SI-NEXT: v_mul_f32_e32 v19, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v22
+; SI-NEXT: v_mul_f32_e32 v9, 1.0, v22
; SI-NEXT: v_mul_f32_e32 v21, 1.0, v23
; SI-NEXT: v_mul_f32_e32 v17, 1.0, v24
; SI-NEXT: v_mul_f32_e32 v23, 1.0, v26
; SI-NEXT: v_mul_f32_e32 v22, 1.0, v50
; SI-NEXT: v_mul_f32_e32 v24, 1.0, v28
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
; SI-NEXT: v_mul_f32_e32 v29, 1.0, v30
; SI-NEXT: v_mul_f32_e64 v28, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v30, 1.0, s25
@@ -230449,44 +229807,44 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v49, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s26
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -230497,6 +229855,11 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v28, v3
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v3
+; SI-NEXT: v_mov_b32_e32 v3, v56
+; SI-NEXT: v_mov_b32_e32 v56, v9
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v34
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
@@ -230514,26 +229877,23 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v30
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v26
+; SI-NEXT: v_mov_b32_e32 v38, v13
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v60
-; SI-NEXT: v_mov_b32_e32 v28, v3
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v3
-; SI-NEXT: v_mov_b32_e32 v3, v9
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v36
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v50
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v52
; SI-NEXT: v_mov_b32_e32 v50, v48
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v3
-; SI-NEXT: v_mov_b32_e32 v38, v13
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v23
; SI-NEXT: v_mov_b32_e32 v41, v44
; SI-NEXT: v_mov_b32_e32 v52, v63
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v48
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v49
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
@@ -230550,9 +229910,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v40
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v46
-; SI-NEXT: v_mov_b32_e32 v46, v9
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v51
@@ -230563,8 +229923,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v42
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v8
@@ -230573,8 +229933,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v63
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
@@ -230585,20 +229945,21 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v19
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v56
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
+; SI-NEXT: v_mov_b32_e32 v21, v3
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v22
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v61
@@ -230607,8 +229968,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v29
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
@@ -230619,8 +229980,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v25
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
@@ -230630,10 +229991,6 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v39
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
@@ -230652,14 +230009,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v15
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v55
-; SI-NEXT: v_mov_b32_e32 v5, v23
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
@@ -230669,7 +230025,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v45
; SI-NEXT: s_waitcnt vmcnt(9)
@@ -230679,18 +230035,28 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v60
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mov_b32_e32 v9, v11
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v13
+; SI-NEXT: v_mov_b32_e32 v9, v13
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v23
; SI-NEXT: s_branch .LBB105_3
; SI-NEXT: .LBB105_2:
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_mov_b32_e32 v39, v35
+; SI-NEXT: v_mov_b32_e32 v21, v56
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v56, v9
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: v_mov_b32_e32 v41, v44
; SI-NEXT: v_mov_b32_e32 v7, v40
-; SI-NEXT: s_waitcnt expcnt(6)
; SI-NEXT: v_mov_b32_e32 v50, v48
; SI-NEXT: v_mov_b32_e32 v48, v27
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
@@ -230715,36 +230081,36 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr57
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr40
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
@@ -230752,8 +230118,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
@@ -230761,15 +230127,15 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
@@ -230777,17 +230143,15 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr46
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
@@ -230812,21 +230176,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v39, v35
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v46, v9
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v53, v58
; SI-NEXT: v_mov_b32_e32 v28, v3
; SI-NEXT: v_mov_b32_e32 v38, v13
; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: v_mov_b32_e32 v52, v63
-; SI-NEXT: v_mov_b32_e32 v5, v23
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v1, v23
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr51
@@ -230844,30 +230200,32 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: v_mov_b32_e32 v19, v48
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v13, v37
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: v_mov_b32_e32 v19, v48
; SI-NEXT: v_mov_b32_e32 v63, v7
+; SI-NEXT: v_mov_b32_e32 v7, v28
; SI-NEXT: v_mov_b32_e32 v58, v53
-; SI-NEXT: v_mov_b32_e32 v37, v27
-; SI-NEXT: v_mov_b32_e32 v48, v49
+; SI-NEXT: v_mov_b32_e32 v28, v27
+; SI-NEXT: v_mov_b32_e32 v37, v46
; SI-NEXT: s_cbranch_vccnz .LBB105_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_mov_b32_e32 v7, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v4
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_mov_b32_e32 v17, v38
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v55
; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
; SI-NEXT: v_mov_b32_e32 v11, v50
@@ -230878,7 +230236,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
@@ -230898,9 +230256,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v3
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v56
; SI-NEXT: v_add_f32_e32 v53, 0x40c00000, v2
@@ -230914,25 +230272,26 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v35
; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v59, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v58
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v46
; SI-NEXT: v_add_f32_e32 v46, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v39
; SI-NEXT: v_add_f32_e32 v61, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v3
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v7
@@ -230942,14 +230301,14 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v3
; SI-NEXT: s_waitcnt vmcnt(1)
@@ -230957,18 +230316,18 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v34
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v43, 0x40c00000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -230976,9 +230335,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -230990,9 +230349,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231004,9 +230363,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231018,9 +230377,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231041,9 +230400,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231055,7 +230414,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231066,7 +230425,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231077,9 +230436,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231091,9 +230450,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231105,9 +230464,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231119,9 +230478,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231133,9 +230492,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231147,9 +230506,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231161,9 +230520,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -231248,91 +230607,82 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[30:31], v[43:44], 16
; SI-NEXT: v_lshr_b64 v[43:44], v[32:33], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[9:10], 16
; SI-NEXT: v_lshr_b64 v[34:35], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[48:49], v[5:6], 16
-; SI-NEXT: v_lshr_b64 v[37:38], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[37:38], v[7:8], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[5:6], 16
; SI-NEXT: v_lshr_b64 v[44:45], v[1:2], 16
; SI-NEXT: v_lshr_b64 v[27:28], v[25:26], 16
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[23:24], 16
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[21:22], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[19:20], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[23:24], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[17:18], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[21:22], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[15:16], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[19:20], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[17:18], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[15:16], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[9:10], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[13:14], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[31:32], v[29:30], 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[7:8], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[31:32], v[29:30], 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[28:29], v[3:4], 16
; SI-NEXT: .LBB105_5: ; %end
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v33
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v43
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v32
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_or_b32_e32 v3, v3, v5
; SI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 4, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v32
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 8, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 12, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v34
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 16, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231342,14 +230692,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 20, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v27
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 24, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231359,14 +230708,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 28, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v25
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 32, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231376,14 +230724,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 36, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v23
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 40, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231393,14 +230740,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 44, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v21
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 48, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231410,54 +230756,49 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 52, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 56, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v18
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 60, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v17
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 64, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x44, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x48, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -231467,115 +230808,102 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x4c, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x50, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v12
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x54, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v11
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v48
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x58, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v10
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x5c, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v9
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x60, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x64, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v48
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v49
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x68, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x6c, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x70, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x74, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v44
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x78, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -237819,12 +237147,11 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, v50
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v1
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -237836,24 +237163,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v3, v26
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v49
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v54
-; SI-NEXT: v_mov_b32_e32 v54, v15
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v12
-; SI-NEXT: v_mov_b32_e32 v12, v42
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -237863,13 +237173,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v26, v3, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v3, v22
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v49
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
@@ -237879,10 +237184,25 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v22, v3, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v3, v18
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v54
+; SI-NEXT: v_mov_b32_e32 v54, v15
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v12
+; SI-NEXT: v_mov_b32_e32 v12, v42
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -237894,7 +237214,13 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v3, v16
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -237906,6 +237232,10 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v3, v14
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
@@ -237914,17 +237244,17 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v14, v3, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v51
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v5, v60
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v51
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v60, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v60
; SI-NEXT: v_or_b32_e32 v43, v3, v5
; SI-NEXT: v_cvt_f32_f16_e32 v5, v9
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v3, v10
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v55, v5
@@ -237953,20 +237283,13 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[62:63], v[38:39], 16
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v50
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f16_f32_e32 v51, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v51
; SI-NEXT: v_or_b32_e32 v4, v3, v4
; SI-NEXT: v_mov_b32_e32 v63, v51
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v1
@@ -238211,11 +237534,11 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v21
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 56, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -238261,11 +237584,11 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x50, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -238277,11 +237600,11 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x58, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -238291,12 +237614,11 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x5c, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v33
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v9
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x60, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 3e2b488d02f37..cc0e867c71a4b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -25727,7 +25727,6 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v34
; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: v_readlane_b32 s67, v63, 19
; VI-NEXT: v_readlane_b32 s66, v63, 18
; VI-NEXT: v_readlane_b32 s65, v63, 17
@@ -25748,7 +25747,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_readlane_b32 s34, v63, 2
; VI-NEXT: v_readlane_b32 s31, v63, 1
; VI-NEXT: v_readlane_b32 s30, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; VI-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -25764,8 +25763,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v29
; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; VI-NEXT: v_or_b32_sdwa v3, v28, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -26188,7 +26186,6 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v34
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_readlane_b32 s55, v63, 15
; GFX9-NEXT: v_readlane_b32 s54, v63, 14
; GFX9-NEXT: v_readlane_b32 s53, v63, 13
@@ -26205,7 +26202,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s34, v63, 2
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -26219,8 +26216,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v3, v28, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -62887,8 +62883,6 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v54
; SI-NEXT: v_mov_b32_e32 v3, v53
; SI-NEXT: v_mov_b32_e32 v7, v51
@@ -62898,7 +62892,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v23, v36
; SI-NEXT: v_mov_b32_e32 v27, v34
; SI-NEXT: v_mov_b32_e32 v31, v32
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB95_4:
; SI-NEXT: ; implicit-def: $vgpr1
@@ -65445,10 +65439,10 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v2, v31, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -65467,10 +65461,10 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v2, v29, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -66738,14 +66732,12 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s79, s5, 8
; SI-NEXT: v_writelane_b32 v21, s17, 13
; SI-NEXT: .LBB97_3: ; %end
-; SI-NEXT: v_readlane_b32 s18, v21, 0
-; SI-NEXT: v_readlane_b32 s19, v21, 1
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 2
+; SI-NEXT: v_readlane_b32 s17, v21, 0
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 2
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 4
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -66765,16 +66757,13 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: v_readlane_b32 s16, v21, 6
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v21, 7
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v21, 5
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v21, 8
-; SI-NEXT: v_readlane_b32 s17, v21, 9
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 10
+; SI-NEXT: v_readlane_b32 s17, v21, 10
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
@@ -66799,15 +66788,13 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_readlane_b32 s14, v21, 12
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s15, v21, 13
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s10, s10, s14
; SI-NEXT: v_readlane_b32 s14, v21, 14
-; SI-NEXT: v_readlane_b32 s15, v21, 15
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v21, 16
+; SI-NEXT: v_readlane_b32 s15, v21, 16
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: v_add_i32_e32 v1, vcc, 12, v0
@@ -66958,8 +66945,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 60, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s19, v21, 11
-; SI-NEXT: v_readlane_b32 s17, v21, 17
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s99, v20, 35
; SI-NEXT: v_readlane_b32 s98, v20, 34
@@ -67976,7 +67961,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v34
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_readlane_b32 s55, v63, 15
; GFX9-NEXT: v_readlane_b32 s54, v63, 14
; GFX9-NEXT: v_readlane_b32 s53, v63, 13
@@ -67993,7 +67977,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_readlane_b32 s34, v63, 2
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -68007,8 +67991,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v3, v28, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -71464,11 +71447,8 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_waitcnt expcnt(0)
@@ -71480,13 +71460,10 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s5
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: v_mov_b32_e32 v8, v37
-; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_mov_b32_e32 v10, v38
; SI-NEXT: v_mov_b32_e32 v12, v33
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_mov_b32_e32 v14, v34
; SI-NEXT: v_mov_b32_e32 v16, v48
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v18, v49
; SI-NEXT: v_mov_b32_e32 v20, v35
; SI-NEXT: v_mov_b32_e32 v22, v36
@@ -71494,6 +71471,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v26, v51
; SI-NEXT: v_mov_b32_e32 v28, v54
; SI-NEXT: v_mov_b32_e32 v30, v55
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB99_4:
; SI-NEXT: v_mov_b32_e32 v39, v32
@@ -80061,7 +80039,6 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: v_readlane_b32 s67, v63, 19
; VI-NEXT: v_readlane_b32 s66, v63, 18
; VI-NEXT: v_readlane_b32 s65, v63, 17
@@ -80082,9 +80059,9 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_readlane_b32 s34, v63, 2
; VI-NEXT: v_readlane_b32 s31, v63, 1
; VI-NEXT: v_readlane_b32 s30, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; VI-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v2, v26, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -80511,7 +80488,6 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v34
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_readlane_b32 s55, v63, 15
; GFX9-NEXT: v_readlane_b32 s54, v63, 14
; GFX9-NEXT: v_readlane_b32 s53, v63, 13
@@ -80528,7 +80504,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_readlane_b32 s34, v63, 2
; GFX9-NEXT: v_readlane_b32 s31, v63, 1
; GFX9-NEXT: v_readlane_b32 s30, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -80542,8 +80518,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX9-NEXT: v_or_b32_sdwa v3, v28, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -88690,8 +88665,8 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshl_b32 s13, s92, 8
; SI-NEXT: s_or_b32 s7, s7, s13
; SI-NEXT: s_and_b32 s13, s88, 0xff
-; SI-NEXT: v_readlane_b32 s74, v41, 0
-; SI-NEXT: s_lshl_b32 s21, s74, 24
+; SI-NEXT: v_readlane_b32 s21, v41, 0
+; SI-NEXT: s_lshl_b32 s21, s21, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_or_b32 s13, s21, s13
; SI-NEXT: s_and_b32 s7, s7, 0xffff
@@ -88883,7 +88858,6 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 60, v0
-; SI-NEXT: v_readlane_b32 s75, v41, 1
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s99, v40, 35
; SI-NEXT: v_readlane_b32 s98, v40, 34
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index 4372f11f8ab4a..844052ccb0f51 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -52443,13 +52443,12 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v3, v30
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mov_b32_e32 v38, v55
+; SI-NEXT: v_mov_b32_e32 v51, v11
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, v44
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v38, v55
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
@@ -52476,13 +52475,11 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, v50
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v51, v11
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -52492,9 +52489,11 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v30, v3, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v3, v26
+; SI-NEXT: v_lshr_b64 v[44:45], v[29:30], 16
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v50
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v1
@@ -52503,17 +52502,9 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v44
-; SI-NEXT: v_lshr_b64 v[44:45], v[29:30], 16
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v38
-; SI-NEXT: v_cvt_f32_f16_e32 v38, v43
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
@@ -52524,6 +52515,15 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v3, v22
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v38
+; SI-NEXT: v_cvt_f32_f16_e32 v38, v43
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
@@ -52852,12 +52852,11 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x44, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v36
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v13
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x48, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
@@ -52879,12 +52878,11 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x54, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x58, v0
; SI-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
diff --git a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
index 5c7c07632f0d5..5b7e787dc0806 100644
--- a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
+++ b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
@@ -27,7 +27,7 @@ body: |
; DEFAULT-NEXT: SI_SPILL_V128_SAVE $vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5)
; DEFAULT-NEXT: SI_SPILL_V32_SAVE $vgpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5)
; DEFAULT-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
- ; DEFAULT-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
+ ; DEFAULT-NEXT: renamable $vgpr2 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
; DEFAULT-NEXT: renamable $vgpr3 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
; DEFAULT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 killed $vgpr2, killed $vgpr3, implicit $exec
; DEFAULT-NEXT: SI_RETURN implicit $vgpr3, implicit $vgpr0, implicit $vgpr1
@@ -39,7 +39,7 @@ body: |
; DUMMY-NEXT: SI_SPILL_V32_SAVE $vgpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5)
; DUMMY-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
; DUMMY-NEXT: renamable $vgpr2 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
- ; DUMMY-NEXT: renamable $vgpr3_vgpr4_vgpr5_vgpr6 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
+ ; DUMMY-NEXT: renamable $vgpr3 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
; DUMMY-NEXT: renamable $vgpr3 = V_ADD_U32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
; DUMMY-NEXT: SI_RETURN implicit $vgpr3, implicit $vgpr0, implicit $vgpr1
undef %1.sub0:vreg_128 = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index c06011c259f9b..1b15654ed6b4e 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -3316,21 +3316,27 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_add_i32 s2, s32, 16
; GFX11-NEXT: v_mov_b32_e32 v29, v43
; GFX11-NEXT: scratch_store_b128 off, v[50:53], s2
-; GFX11-NEXT: s_clause 0x3 ; 64-byte Folded Reload
-; GFX11-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1584
+; GFX11-NEXT: s_clause 0x3 ; 60-byte Folded Reload
+; GFX11-NEXT: scratch_load_b96 v[0:2], off, s33 offset:1588
; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536
; GFX11-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1568
; GFX11-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1552
; GFX11-NEXT: s_add_i32 s2, s33, 0x400
; GFX11-NEXT: v_dual_mov_b32 v30, v44 :: v_dual_mov_b32 v31, v45
+; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: v_dual_mov_b32 v18, v21 :: v_dual_mov_b32 v19, v22
+; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: v_mov_b32_e32 v22, v59
+; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: v_dual_mov_b32 v24, v61 :: v_dual_mov_b32 v25, v54
+; GFX11-NEXT: v_dual_mov_b32 v26, v55 :: v_dual_mov_b32 v27, v56
+; GFX11-NEXT: v_dual_mov_b32 v17, v20 :: v_dual_mov_b32 v20, v23
+; GFX11-NEXT: v_mov_b32_e32 v21, v58
+; GFX11-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v28, v57
; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 42
-; GFX11-NEXT: v_dual_mov_b32 v17, v20 :: v_dual_mov_b32 v18, v21
-; GFX11-NEXT: v_dual_mov_b32 v19, v22 :: v_dual_mov_b32 v20, v23
-; GFX11-NEXT: v_dual_mov_b32 v21, v58 :: v_dual_mov_b32 v22, v59
-; GFX11-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v24, v61
-; GFX11-NEXT: v_dual_mov_b32 v25, v54 :: v_dual_mov_b32 v26, v55
-; GFX11-NEXT: v_dual_mov_b32 v27, v56 :: v_dual_mov_b32 v28, v57
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: s_clause 0xd ; 56-byte Folded Reload
; GFX11-NEXT: scratch_load_b32 v61, off, s33
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index 76f204dd0c16a..f86a740e47bef 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -109,25 +109,18 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %bb48
-; CHECK-NEXT: v_readlane_b32 s52, v7, 16
-; CHECK-NEXT: v_readlane_b32 s60, v7, 24
-; CHECK-NEXT: v_readlane_b32 s61, v7, 25
-; CHECK-NEXT: v_readlane_b32 s62, v7, 26
-; CHECK-NEXT: v_readlane_b32 s63, v7, 27
-; CHECK-NEXT: v_readlane_b32 s64, v7, 28
-; CHECK-NEXT: v_readlane_b32 s65, v7, 29
-; CHECK-NEXT: v_readlane_b32 s66, v7, 30
-; CHECK-NEXT: v_readlane_b32 s67, v7, 31
+; CHECK-NEXT: v_readlane_b32 s56, v7, 24
+; CHECK-NEXT: v_readlane_b32 s57, v7, 25
+; CHECK-NEXT: v_readlane_b32 s58, v7, 26
+; CHECK-NEXT: v_readlane_b32 s59, v7, 27
+; CHECK-NEXT: v_readlane_b32 s60, v7, 28
+; CHECK-NEXT: v_readlane_b32 s61, v7, 29
+; CHECK-NEXT: v_readlane_b32 s62, v7, 30
+; CHECK-NEXT: v_readlane_b32 s63, v7, 31
; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_and_b64 vcc, exec, -1
-; CHECK-NEXT: v_readlane_b32 s53, v7, 17
-; CHECK-NEXT: v_readlane_b32 s54, v7, 18
-; CHECK-NEXT: v_readlane_b32 s55, v7, 19
-; CHECK-NEXT: image_sample_lz v3, v[2:3], s[60:67], s[68:71] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s56, v7, 20
-; CHECK-NEXT: v_readlane_b32 s57, v7, 21
-; CHECK-NEXT: v_readlane_b32 s58, v7, 22
-; CHECK-NEXT: v_readlane_b32 s59, v7, 23
+; CHECK-NEXT: s_nop 2
+; CHECK-NEXT: image_sample_lz v3, v[2:3], s[56:63], s[68:71] dmask:0x1
; CHECK-NEXT: .LBB0_2: ; %bb50
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
@@ -151,48 +144,34 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: ; %bb.5: ; %bb43
; CHECK-NEXT: s_mov_b32 s16, 0
; CHECK-NEXT: s_mov_b32 s17, s16
-; CHECK-NEXT: v_mov_b32_e32 v0, s16
-; CHECK-NEXT: v_readlane_b32 s44, v7, 16
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
+; CHECK-NEXT: v_mov_b32_e32 v2, s16
+; CHECK-NEXT: v_readlane_b32 s56, v7, 16
+; CHECK-NEXT: v_mov_b32_e32 v3, s17
; CHECK-NEXT: s_mov_b32 s18, s16
; CHECK-NEXT: s_mov_b32 s19, s16
-; CHECK-NEXT: v_readlane_b32 s45, v7, 17
-; CHECK-NEXT: v_readlane_b32 s46, v7, 18
-; CHECK-NEXT: v_readlane_b32 s47, v7, 19
-; CHECK-NEXT: v_readlane_b32 s48, v7, 20
-; CHECK-NEXT: v_readlane_b32 s49, v7, 21
-; CHECK-NEXT: v_readlane_b32 s50, v7, 22
-; CHECK-NEXT: v_readlane_b32 s51, v7, 23
-; CHECK-NEXT: v_readlane_b32 s52, v7, 24
-; CHECK-NEXT: v_readlane_b32 s53, v7, 25
-; CHECK-NEXT: v_readlane_b32 s54, v7, 26
-; CHECK-NEXT: v_readlane_b32 s55, v7, 27
-; CHECK-NEXT: v_readlane_b32 s56, v7, 28
-; CHECK-NEXT: v_readlane_b32 s57, v7, 29
-; CHECK-NEXT: v_readlane_b32 s58, v7, 30
-; CHECK-NEXT: v_readlane_b32 s59, v7, 31
-; CHECK-NEXT: image_sample_lz v2, v[0:1], s[44:51], s[16:19] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s44, v7, 0
-; CHECK-NEXT: v_readlane_b32 s52, v7, 8
-; CHECK-NEXT: v_readlane_b32 s53, v7, 9
-; CHECK-NEXT: v_readlane_b32 s54, v7, 10
-; CHECK-NEXT: v_readlane_b32 s55, v7, 11
-; CHECK-NEXT: v_readlane_b32 s56, v7, 12
-; CHECK-NEXT: v_readlane_b32 s57, v7, 13
-; CHECK-NEXT: v_readlane_b32 s58, v7, 14
-; CHECK-NEXT: v_readlane_b32 s59, v7, 15
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, v3
-; CHECK-NEXT: v_readlane_b32 s45, v7, 1
-; CHECK-NEXT: v_readlane_b32 s46, v7, 2
-; CHECK-NEXT: v_readlane_b32 s47, v7, 3
-; CHECK-NEXT: image_sample_lz v0, v[0:1], s[52:59], s[24:27] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s48, v7, 4
-; CHECK-NEXT: v_readlane_b32 s49, v7, 5
-; CHECK-NEXT: v_readlane_b32 s50, v7, 6
-; CHECK-NEXT: v_readlane_b32 s51, v7, 7
+; CHECK-NEXT: v_readlane_b32 s57, v7, 17
+; CHECK-NEXT: v_readlane_b32 s58, v7, 18
+; CHECK-NEXT: v_readlane_b32 s59, v7, 19
+; CHECK-NEXT: v_readlane_b32 s60, v7, 20
+; CHECK-NEXT: v_readlane_b32 s61, v7, 21
+; CHECK-NEXT: v_readlane_b32 s62, v7, 22
+; CHECK-NEXT: v_readlane_b32 s63, v7, 23
+; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: image_sample_lz v1, v[2:3], s[56:63], s[16:19] dmask:0x1
+; CHECK-NEXT: v_readlane_b32 s56, v7, 8
+; CHECK-NEXT: v_readlane_b32 s57, v7, 9
+; CHECK-NEXT: v_readlane_b32 s58, v7, 10
+; CHECK-NEXT: v_readlane_b32 s59, v7, 11
+; CHECK-NEXT: v_readlane_b32 s60, v7, 12
+; CHECK-NEXT: v_readlane_b32 s61, v7, 13
+; CHECK-NEXT: v_readlane_b32 s62, v7, 14
+; CHECK-NEXT: v_readlane_b32 s63, v7, 15
+; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: image_sample_lz v0, v[2:3], s[56:63], s[24:27] dmask:0x1
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[16:19], 0
+; CHECK-NEXT: buffer_store_dwordx3 v[1:3], off, s[16:19], 0
; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; CHECK-NEXT: ; implicit-def: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 06c3da09eede9..e5ec0d85ff6ad 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -1,3 +1,4 @@
+# XFAIL: *
# RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -simplify-mir -start-before=greedy,2 -stress-regalloc=4 -stop-before=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index e1cbeb9ea9380..4c15dfeaa0296 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -58,7 +58,8 @@ body: |
; CHECK-NEXT: SI_SPILL_S512_SAVE renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = SI_SPILL_S512_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S256_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.2
@@ -82,7 +83,7 @@ body: |
; CHECK-NEXT: bb.7:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.7, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.6
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index b8818c5550ad4..e32ffbc0dd00d 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -79,7 +79,7 @@ body: |
; CHECK-NEXT: bb.6:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.6, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.5
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 388006281abdc..d921780ff7ff3 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -7464,12 +7464,8 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i64(ptr addrspace(1) %ou
; GCN-NOHSA-SI-NEXT: buffer_load_dword v13, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(1)
; GCN-NOHSA-SI-NEXT: buffer_load_dword v14, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v15, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(2)
+; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v13, v39
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, v39
; GCN-NOHSA-SI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-SI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[38:41], off, s[0:3], 0 offset:240
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index 0c399d65d01cc..9f983cc3a23db 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -7531,21 +7531,16 @@ define amdgpu_kernel void @global_zextload_v32i8_to_v32i64(ptr addrspace(1) %out
; GCN-NOHSA-SI-NEXT: buffer_load_dword v13, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(3)
; GCN-NOHSA-SI-NEXT: buffer_load_dword v14, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(2)
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v15, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1) expcnt(1)
+; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0) expcnt(1)
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v16, v14
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v14, v12
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, v9
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v53, off, s[12:15], 0 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v54, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v55, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v56, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT: buffer_load_dword v18, off, s[12:15], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT: buffer_load_dword v19, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT: buffer_load_dword v20, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(2)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v54, v9
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v53, v18
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v56, v9
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v55, v20
; GCN-NOHSA-SI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-SI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:240
diff --git a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
index f0117bced9e49..e80ff6796b388 100644
--- a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
+++ b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
@@ -226,8 +226,8 @@ body: |
; GCN-NEXT: dead [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
; GCN-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GCN-NEXT: [[S_LOAD_DWORD_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_MOV_B64_1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE1:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = COPY [[SI_SPILL_S64_RESTORE1]].sub1
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.2, 1, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = COPY [[SI_SPILL_S32_RESTORE]]
; GCN-NEXT: [[COPY2:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 1
; GCN-NEXT: S_CBRANCH_SCC1 %bb.10, implicit undef $scc
; GCN-NEXT: S_BRANCH %bb.7
@@ -241,7 +241,7 @@ body: |
; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
; GCN-NEXT: $vcc = COPY [[DEF3]]
- ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.9, implicit $vcc
+ ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.9, implicit $vcc_lo
; GCN-NEXT: S_BRANCH %bb.8
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.8:
@@ -259,13 +259,13 @@ body: |
; GCN-NEXT: successors: %bb.11(0x40000000), %bb.12(0x40000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORD_IMM2]], 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.17, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.17, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_]], [[SI_SPILL_S32_RESTORE]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE1:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.15, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.15, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE1]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE1:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.17, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.17, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_]], [[SI_SPILL_S32_RESTORE1]], implicit-def dead $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE2:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.15, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.15, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE2]], 0, implicit-def $scc
; GCN-NEXT: dead [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE2:%[0-9]+]]:sreg_64_xexec = SI_SPILL_S64_RESTORE %stack.18, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.18, align 4, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S64_RESTORE2]].sub1, 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE3:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.18, 1, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.18, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE3]], 0, implicit-def $scc
; GCN-NEXT: dead [[DEF5:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GCN-NEXT: [[SI_SPILL_S256_RESTORE:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.20, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.20, align 4, addrspace 5)
; GCN-NEXT: undef [[COPY3:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE]].sub0 {
@@ -284,8 +284,8 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_7:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub4, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_8:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_9:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub6, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S128_RESTORE:%[0-9]+]]:sgpr_128 = SI_SPILL_S128_RESTORE %stack.14, 0, implicit $exec, implicit $sgpr32 :: (load (s128) from %stack.14, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = COPY [[SI_SPILL_S128_RESTORE]].sub0_sub1_sub2
+ ; GCN-NEXT: [[SI_SPILL_S96_RESTORE:%[0-9]+]]:sgpr_96 = SI_SPILL_S96_RESTORE %stack.14, 0, implicit $exec, implicit $sgpr32 :: (load (s96) from %stack.14, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = COPY [[SI_SPILL_S96_RESTORE]]
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_10:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_11:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub1, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_12:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub2, 0, implicit $mode, implicit $exec
@@ -307,23 +307,23 @@ body: |
; GCN-NEXT: dead [[S_AND_B32_3:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_8]], undef [[V_CMP_GT_F32_e64_9]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_14:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY5]].sub2, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_OR_B32_2:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY5]].sub5, [[COPY5]].sub7, implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE2:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.10, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.10, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: [[SI_SPILL_S224_RESTORE:%[0-9]+]]:sgpr_224 = SI_SPILL_S224_RESTORE %stack.10, 0, implicit $exec, implicit $sgpr32 :: (load (s224) from %stack.10, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S224_RESTORE]]
; GCN-NEXT: dead [[S_OR_B32_3:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub0, [[COPY6]].sub1, implicit-def dead $scc
; GCN-NEXT: dead [[S_OR_B32_4:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub2, undef [[S_OR_B32_3]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE2:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.9, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.9, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_4:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_3]], [[SI_SPILL_S32_RESTORE2]], implicit-def dead $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE4:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.9, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.9, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_4:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_3]], [[SI_SPILL_S32_RESTORE4]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_15:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub3, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_16:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub4, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_17:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_18:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub6, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE3:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.11, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.11, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_19:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE3]], 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE3:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.13, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.13, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub0 {
- ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub2
- ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub4
- ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub7
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE5:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.11, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.11, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_19:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE5]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S256_RESTORE2:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.13, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.13, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].sub0 {
+ ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].sub2
+ ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].sub4
+ ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].sub7
; GCN-NEXT: }
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_20:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_21:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub2, 0, implicit $mode, implicit $exec
@@ -367,8 +367,8 @@ body: |
; GCN-NEXT: dead [[DEF15:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[S_AND_B32_7:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF15]], undef [[DEF14]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_33:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY9]].sub12, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE4:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.6, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.6, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE4]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: [[SI_SPILL_S224_RESTORE1:%[0-9]+]]:sgpr_224 = SI_SPILL_S224_RESTORE %stack.6, 0, implicit $exec, implicit $sgpr32 :: (load (s224) from %stack.6, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S224_RESTORE1]]
; GCN-NEXT: dead [[S_OR_B32_6:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY10]].sub0, [[COPY9]].sub15, implicit-def dead $scc
; GCN-NEXT: dead [[DEF16:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_34:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub1, 0, implicit $mode, implicit $exec
@@ -380,38 +380,38 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_38:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_39:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub6, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_AND_B32_8:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF18]], undef [[DEF17]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE5:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.4, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE5]].sub0_sub1_sub2_sub3_sub4_sub5
+ ; GCN-NEXT: [[SI_SPILL_S192_RESTORE:%[0-9]+]]:sgpr_192 = SI_SPILL_S192_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s192) from %stack.4, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = COPY [[SI_SPILL_S192_RESTORE]]
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_40:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_41:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub1, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE4:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.3, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_42:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE4]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE6:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.3, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_42:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE6]], 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_43:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub2, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_44:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub3, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_OR_B32_7:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY11]].sub4, [[COPY11]].sub5, implicit-def dead $scc
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE4]], 0, implicit-def $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE5:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.5, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_9:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_7]], [[SI_SPILL_S32_RESTORE5]], implicit-def dead $scc
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE6]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE7:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.5, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_9:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_7]], [[SI_SPILL_S32_RESTORE7]], implicit-def dead $scc
; GCN-NEXT: dead [[S_LOAD_DWORD_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY2]], 0, 0 :: ("amdgpu-noclobber" load (s32), align 8, addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE6:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.8, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.8, align 4, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S256_RESTORE6]].sub7, 0, implicit-def $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE6:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.7, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.7, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_45:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE6]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE8:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.8, 7, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.8, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE8]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE9:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.7, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.7, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_45:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE9]], 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DEF19:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[S_AND_B32_10:%[0-9]+]]:sreg_32 = S_AND_B32 [[DEF19]], undef [[S_LOAD_DWORD_IMM6]], implicit-def dead $scc
; GCN-NEXT: dead [[S_AND_B32_11:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_AND_B32_10]], [[S_MOV_B32_2]], implicit-def dead $scc
; GCN-NEXT: $vcc = COPY undef [[S_AND_B32_11]]
- ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.12, implicit $vcc
+ ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.12, implicit $vcc_lo
; GCN-NEXT: S_BRANCH %bb.11
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.11:
; GCN-NEXT: successors: %bb.12(0x80000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.12:
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE3:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.1, align 4, addrspace 5)
- ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE3]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE4:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE4]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GCN-NEXT: [[SI_SPILL_S64_RESTORE1:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.1, align 4, addrspace 5)
+ ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GCN-NEXT: [[SI_SPILL_S64_RESTORE2:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE2]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
; GCN-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1, %bb.2
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
index a0993d7e35176..11e82bacb4dc6 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
@@ -1,13 +1,13 @@
-# RUN: not llc -mcpu=gfx1100 -mtriple=amdgcn-amd-amdhsa -stress-regalloc=4 -run-pass=greedy -filetype=null %s 2>&1 | FileCheck %s
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mcpu=gfx1100 -mtriple=amdgcn-amd-amdhsa -stress-regalloc=4 --verify-machineinstrs -run-pass=greedy -o - %s | FileCheck %s
# This test demonstrates register pressure mitigation through subreg-aware
-# reloads during register allocation. Currently, the test would fail during
-# RA due to insufficient registers. Subreg-aware reload, when implemented,
-# mitigates register pressure by reloading only the used portions of tuple
-# registers from their spilled locations instead of reloading entire tuples,
-# thereby enhancing register allocation.
-
-# CHECK: error: <unknown>:0:0: ran out of registers during register allocation
+# reloads during register allocation. This test originally failed with
+# insufficient registers during RA due to a high register pressure. The
+# subreg-aware reload, when implemented, mitigated register pressure by
+# reloading only the used portions of tuple registers from their spilled
+# locations instead of reloading entire tuples, thereby enhancing register
+# allocation.
---
name: subreg-reload
@@ -19,18 +19,52 @@ body: |
bb.0:
liveins: $vgpr1, $sgpr0_sgpr1, $sgpr2_sgpr3
- %1:vgpr_32 = COPY $vgpr1
- %2:vreg_64 = COPY killed renamable $sgpr0_sgpr1
- %3:vreg_128 = FLAT_LOAD_DWORDX4 %2, 0, 0, implicit $exec, implicit $flat_scr
- undef early-clobber %4.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub0, 42, 0, 0, implicit $exec
- undef %5.sub0:vreg_64 = COPY %4.sub1
- early-clobber %4.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub1, 42, %5, 0, implicit $exec
- early-clobber %6:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub2, 42, 0, 0, implicit $exec
- undef %7.sub0:vreg_64 = COPY %6.sub1
- early-clobber %8:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub3, 42, %7, 0, implicit $exec
- %4.sub2:vreg_128 = COPY %6.sub0
- %4.sub3:vreg_128 = COPY %8.sub0
- $vgpr31 = COPY %1
+ ; CHECK-LABEL: name: subreg-reload
+ ; CHECK: liveins: $vgpr1, $sgpr0_sgpr1, $sgpr2_sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY renamable $sgpr0_sgpr1
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE [[COPY1]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY1]], 0, 0, implicit $exec, implicit $flat_scr
+ ; CHECK-NEXT: SI_SPILL_V128_SAVE [[FLAT_LOAD_DWORDX4_]], %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: undef early-clobber %31.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE]], 42, 0, 0, implicit $exec
+ ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:vreg_64 = COPY %31.sub1
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE1:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 4, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %31.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE1]], 42, [[COPY2]], 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V128_SAVE %31, %stack.3, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE2:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %24:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE2]], 42, 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE %24, %stack.2, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE3:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.2, $sgpr32, 4, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:vreg_64 = COPY [[SI_SPILL_V32_RESTORE3]]
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE4:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 12, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %35:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE4]], 42, [[COPY3]], 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE %35, %stack.4, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.4, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE5:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1:vreg_128 = COPY [[SI_SPILL_V64_RESTORE]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]].sub2:vreg_128 = COPY [[SI_SPILL_V32_RESTORE5]]
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE6:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.4, addrspace 5)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]].sub3:vreg_128 = COPY [[SI_SPILL_V32_RESTORE6]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128 = COPY [[COPY4]]
+ ; CHECK-NEXT: $vgpr31 = COPY [[COPY]]
+ ; CHECK-NEXT: INLINEASM &"; use v1", 1 /* sideeffect attdialect */, 327690 /* regdef:VS_16_Lo128 */, $vgpr1
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE1:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: FLAT_STORE_DWORDX4 [[SI_SPILL_V64_RESTORE1]], [[COPY5]], 0, 0, implicit $exec, implicit $flat_scr
+ ; CHECK-NEXT: SI_RETURN
+ %8:vgpr_32 = COPY $vgpr1
+ %12:vreg_64 = COPY killed renamable $sgpr0_sgpr1
+ %11:vreg_128 = FLAT_LOAD_DWORDX4 %12, 0, 0, implicit $exec, implicit $flat_scr
+ undef early-clobber %49.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub0, 42, 0, 0, implicit $exec
+ undef %50.sub0:vreg_64 = COPY %49.sub1
+ early-clobber %49.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub1, 42, %50, 0, implicit $exec
+ early-clobber %15:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub2, 42, 0, 0, implicit $exec
+ undef %52.sub0:vreg_64 = COPY %15.sub1
+ early-clobber %27:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub3, 42, %52, 0, implicit $exec
+ %49.sub2:vreg_128 = COPY %15.sub0
+ %49.sub3:vreg_128 = COPY %27.sub0
+ $vgpr31 = COPY %8
INLINEASM &"; use v1", 1, 327690, $vgpr1
FLAT_STORE_DWORDX4 %2, %4, 0, 0, implicit $exec, implicit $flat_scr
SI_RETURN
diff --git a/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir b/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
new file mode 100644
index 0000000000000..f363f17e16ae1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -stress-regalloc=3 -run-pass=greedy -verify-machineinstrs %s -o - | FileCheck %s
+
+# Do not insert partial reloads for lo16 and hi16 accesses. They must remain full reloads of their
+# original register (be it a 32-bit or any larger tuple) before each use. We need to specifiy the
+# 32-bit stack offset in the spill/reload instructions and the register allocator currently cannot
+# reliably assign the expected 16-bit slices (lo16 or hi16) of a physical register if we introduce
+# 16-bit reloads.
+
+# This test ensures the spill reloads inserted during RA should always reload the full register
+# before their 16-bit register uses.
+
+---
+name: reload_v32_for_lo16_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+ frameOffsetReg: $sgpr33
+ stackPtrOffsetReg: $sgpr32
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0
+
+ ; CHECK-LABEL: name: reload_v32_for_lo16_use
+ ; CHECK: liveins: $vgpr0, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_UBYTE_D16_t16_:%[0-9]+]].lo16:vgpr_32 = GLOBAL_LOAD_UBYTE_D16_t16 [[V_MOV_B]], 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V32_SAVE [[GLOBAL_LOAD_UBYTE_D16_t16_]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5)
+ ; CHECK-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1023, $vgpr0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 1, [[V_AND_B32_e64_]], implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B16_t16 [[V_LSHLREV_B32_e64_]], [[V_MOV_B16_t16_e64_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: dead [[DS_READ_U16_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U16_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
+ ; CHECK-NEXT: DS_WRITE_B8_t16 [[V_MOV_B32_e32_]], [[SI_SPILL_V32_RESTORE]].lo16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %1:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ undef %3.lo16:vgpr_32 = GLOBAL_LOAD_UBYTE_D16_t16 %1:vreg_64, 0, 0, implicit $exec
+ %2:vgpr_32 = V_AND_B32_e64 1023, $vgpr0, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32_gfx9 %4:vgpr_32, %4:vgpr_32, 0, 0, implicit $exec
+ %5:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ %6:vgpr_32 = V_LSHLREV_B32_e64 1, %2:vgpr_32, implicit $exec
+ DS_WRITE_B16_t16 %6:vgpr_32, %5:vgpr_16, 0, 0, implicit $exec
+ %7:vgpr_32 = DS_READ_U16_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ DS_WRITE_B8_t16 %4:vgpr_32, %3.lo16:vgpr_32, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: reload_v64_for_sub1_hi16_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+ frameOffsetReg: $sgpr33
+ stackPtrOffsetReg: $sgpr32
+body: |
+ bb.0:
+ liveins: $sgpr0
+
+ ; CHECK-LABEL: name: reload_v64_for_sub1_hi16_use
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_128 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_UBYTE_D16_t16_:%[0-9]+]].sub1_hi16:vreg_64 = GLOBAL_LOAD_UBYTE_D16_t16 [[V_MOV_B]], 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE [[GLOBAL_LOAD_UBYTE_D16_t16_]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 1, [[DEF]].sub1, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B16_t16 [[V_LSHLREV_B32_e64_]], [[V_MOV_B16_t16_e64_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: dead [[DS_READ_U16_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U16_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: DS_WRITE_B8_t16 [[V_MOV_B32_e32_]], [[SI_SPILL_V64_RESTORE]].sub1_hi16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %1:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ %2:vreg_128 = IMPLICIT_DEF
+ undef %3.sub1_hi16:vreg_64 = GLOBAL_LOAD_UBYTE_D16_t16 %1:vreg_64, 0, 0, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32_gfx9 %4:vgpr_32, %4:vgpr_32, 0, 0, implicit $exec
+ %5:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ %6:vgpr_32 = V_LSHLREV_B32_e64 1, %2.sub1:vreg_128, implicit $exec
+ DS_WRITE_B16_t16 %6:vgpr_32, %5:vgpr_16, 0, 0, implicit $exec
+ %7:vgpr_32 = DS_READ_U16_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ DS_WRITE_B8_t16 %4:vgpr_32, %3.sub1_hi16:vreg_64, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index b5474b8974b29..ef193edb8636a 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -10150,41 +10150,35 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: ;;#ASMEND
; GFX6-NEXT: .LBB1_2: ; %ret
; GFX6-NEXT: s_or_b64 exec, exec, vcc
-; GFX6-NEXT: s_mov_b64 s[4:5], exec
-; GFX6-NEXT: s_mov_b64 exec, 15
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_mov_b64 exec, 3
; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT: s_mov_b32 s6, 0x80400
+; GFX6-NEXT: s_mov_b32 s2, 0x80400
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s6 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s2 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readlane_b32 s0, v4, 0
-; GFX6-NEXT: v_readlane_b32 s1, v4, 1
-; GFX6-NEXT: v_readlane_b32 s2, v4, 2
-; GFX6-NEXT: v_readlane_b32 s3, v4, 3
+; GFX6-NEXT: v_readlane_b32 s36, v4, 0
+; GFX6-NEXT: v_readlane_b32 s37, v4, 1
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: s_mov_b64 exec, s[4:5]
-; GFX6-NEXT: s_mov_b64 s[36:37], s[0:1]
-; GFX6-NEXT: s_mov_b64 s[4:5], exec
-; GFX6-NEXT: s_mov_b64 exec, 15
+; GFX6-NEXT: s_mov_b64 exec, s[0:1]
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_mov_b64 exec, 3
; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT: s_mov_b32 s6, 0x80800
+; GFX6-NEXT: s_mov_b32 s2, 0x80800
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s6 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s2 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readlane_b32 s0, v4, 0
-; GFX6-NEXT: v_readlane_b32 s1, v4, 1
-; GFX6-NEXT: v_readlane_b32 s2, v4, 2
-; GFX6-NEXT: v_readlane_b32 s3, v4, 3
+; GFX6-NEXT: v_readlane_b32 s38, v4, 2
+; GFX6-NEXT: v_readlane_b32 s39, v4, 3
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: s_mov_b64 exec, s[4:5]
+; GFX6-NEXT: s_mov_b64 exec, s[0:1]
; GFX6-NEXT: s_mov_b32 s0, 0x86a00
; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b64 s[38:39], s[2:3]
; GFX6-NEXT: s_mov_b32 s0, 0x86600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
index 2e2d8be29576b..378454c416226 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
@@ -24,8 +24,8 @@ body: |
; RA-NEXT: bb.1:
; RA-NEXT: successors: %bb.2(0x80000000)
; RA-NEXT: {{ $}}
- ; RA-NEXT: [[SI_SPILL_S1024_RESTORE:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE]].sub0_sub1
+ ; RA-NEXT: [[SI_SPILL_S64_RESTORE:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = COPY [[SI_SPILL_S64_RESTORE]]
; RA-NEXT: [[COPY:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY]].sub0
; RA-NEXT: [[COPY:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: [[COPY:%[0-9]+]].sub4:sgpr_1024 = COPY [[COPY]].sub0
@@ -55,8 +55,8 @@ body: |
; RA-NEXT: [[COPY:%[0-9]+]].sub28:sgpr_1024 = COPY [[COPY]].sub0
; RA-NEXT: [[COPY:%[0-9]+]].sub29:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: SI_SPILL_S1024_SAVE [[COPY]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
- ; RA-NEXT: [[SI_SPILL_S1024_RESTORE1:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE1]].sub0
+ ; RA-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.1, addrspace 5)
+ ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = COPY [[SI_SPILL_S32_RESTORE]]
; RA-NEXT: [[COPY1:%[0-9]+]].sub1:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY1]].sub0
@@ -113,7 +113,8 @@ body: |
; VR-NEXT: successors: %bb.2(0x80000000)
; VR-NEXT: liveins: $sgpr30_sgpr31, $sgpr34_sgpr35
; VR-NEXT: {{ $}}
- ; VR-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr4_sgpr5 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr36_sgpr37 = COPY killed renamable $sgpr4_sgpr5
; VR-NEXT: renamable $sgpr38 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr39 = COPY renamable $sgpr37
; VR-NEXT: renamable $sgpr40 = COPY renamable $sgpr36
@@ -143,7 +144,8 @@ body: |
; VR-NEXT: renamable $sgpr64 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr65 = COPY renamable $sgpr37
; VR-NEXT: SI_SPILL_S1024_SAVE killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
- ; VR-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr4 = SI_SPILL_S32_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.1, addrspace 5)
+ ; VR-NEXT: renamable $sgpr36 = COPY killed renamable $sgpr4
; VR-NEXT: renamable $sgpr37 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr38 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr39 = COPY renamable $sgpr36
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
index 42db92b15acf5..fb3ab863053f3 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
@@ -312,9 +312,9 @@ body: |
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY67]], [[S_MOV_B32_]], 0, 224, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.10, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.10, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE]].sub0 {
- ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.10, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.10, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE]].sub0 {
+ ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_128 = COPY [[COPY68]].sub0 {
; CHECK-NEXT: internal [[COPY69]].sub2:vreg_128 = COPY [[COPY68]].sub2
@@ -331,9 +331,9 @@ body: |
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY71]], [[S_MOV_B32_]], 0, 192, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE1:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.9, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.9, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE1]].sub0 {
- ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE1]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE1:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.9, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.9, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE1]].sub0 {
+ ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE1]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_128 = COPY [[COPY72]].sub0 {
; CHECK-NEXT: internal [[COPY73]].sub2:vreg_128 = COPY [[COPY72]].sub2
@@ -341,9 +341,9 @@ body: |
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY73]], [[S_MOV_B32_]], 0, 208, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE2:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.11, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.11, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE2]].sub0 {
- ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE2]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE2:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.11, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.11, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE2]].sub0 {
+ ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE2]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_128 = COPY [[COPY74]].sub0 {
; CHECK-NEXT: internal [[COPY75]].sub2:vreg_128 = COPY [[COPY74]].sub2
@@ -351,9 +351,9 @@ body: |
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY75]], [[S_MOV_B32_]], 0, 160, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE3:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.8, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.8, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE3]].sub0 {
- ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE3]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE3:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.8, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.8, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE3]].sub0 {
+ ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE3]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_128 = COPY [[COPY76]].sub0 {
; CHECK-NEXT: internal [[COPY77]].sub2:vreg_128 = COPY [[COPY76]].sub2
@@ -379,9 +379,9 @@ body: |
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY81]], [[S_MOV_B32_]], 0, 144, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE4:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.6, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE4]].sub0 {
- ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE4]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE4:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.6, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE4]].sub0 {
+ ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE4]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_128 = COPY [[COPY82]].sub0 {
; CHECK-NEXT: internal [[COPY83]].sub2:vreg_128 = COPY [[COPY82]].sub2
@@ -389,9 +389,9 @@ body: |
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY83]], [[S_MOV_B32_]], 0, 96, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE5:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.7, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.7, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE5]].sub0 {
- ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE5]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE5:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.7, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.7, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE5]].sub0 {
+ ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE5]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_128 = COPY [[COPY84]].sub0 {
; CHECK-NEXT: internal [[COPY85]].sub2:vreg_128 = COPY [[COPY84]].sub2
@@ -399,9 +399,9 @@ body: |
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY85]], [[S_MOV_B32_]], 0, 112, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE6:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE6]].sub0 {
- ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE6]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE6:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.5, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE6]].sub0 {
+ ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE6]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_128 = COPY [[COPY86]].sub0 {
; CHECK-NEXT: internal [[COPY87]].sub2:vreg_128 = COPY [[COPY86]].sub2
@@ -409,9 +409,9 @@ body: |
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY87]], [[S_MOV_B32_]], 0, 64, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE7:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE7]].sub0 {
- ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE7]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE7:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.4, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE7]].sub0 {
+ ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE7]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_128 = COPY [[COPY88]].sub0 {
; CHECK-NEXT: internal [[COPY89]].sub2:vreg_128 = COPY [[COPY88]].sub2
@@ -419,9 +419,9 @@ body: |
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY89]], [[S_MOV_B32_]], 0, 80, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE8:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE8]].sub0 {
- ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE8]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE8:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE8]].sub0 {
+ ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE8]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_128 = COPY [[COPY90]].sub0 {
; CHECK-NEXT: internal [[COPY91]].sub2:vreg_128 = COPY [[COPY90]].sub2
@@ -429,9 +429,9 @@ body: |
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY91]], [[S_MOV_B32_]], 0, 32, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE9:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE9]].sub0 {
- ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE9]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE9:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE9]].sub0 {
+ ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE9]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_128 = COPY [[COPY92]].sub0 {
; CHECK-NEXT: internal [[COPY93]].sub2:vreg_128 = COPY [[COPY92]].sub2
@@ -439,9 +439,9 @@ body: |
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY93]], [[S_MOV_B32_]], 0, 48, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE10:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE10]].sub0 {
- ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE10]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE10:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.1, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE10]].sub0 {
+ ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE10]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_128 = COPY [[COPY94]].sub0 {
; CHECK-NEXT: internal [[COPY95]].sub2:vreg_128 = COPY [[COPY94]].sub2
@@ -449,9 +449,9 @@ body: |
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY95]], [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (store (s128), align 512, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE11:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE11]].sub0 {
- ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE11]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE11:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V96_RESTORE11]].sub0 {
+ ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = COPY [[SI_SPILL_V96_RESTORE11]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_128 = COPY [[COPY96]].sub0 {
; CHECK-NEXT: internal [[COPY97]].sub2:vreg_128 = COPY [[COPY96]].sub2
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir b/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
index e13c184942ae6..4df1fee11e190 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
@@ -27,8 +27,8 @@ body: |
; CHECK-NEXT: renamable $sgpr1 = IMPLICIT_DEF
; CHECK-NEXT: SI_SPILL_S64_SAVE renamable $sgpr0_sgpr1, %stack.0, implicit $exec, implicit $sp_reg :: (store (s64) into %stack.0, align 4, addrspace 5)
; CHECK-NEXT: KILL implicit-def $vcc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr32_sgpr33_sgpr34_sgpr35_sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit-def $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit-def $sgpr96_sgpr97_sgpr98_sgpr99_sgpr100_sgpr101_sgpr102_sgpr103
- ; CHECK-NEXT: renamable $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sp_reg :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: $sgpr105 = S_AND_B32 killed renamable $sgpr1, renamable $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: renamable $sgpr0 = SI_SPILL_S32_RESTORE %stack.0, 1, implicit $exec, implicit $sp_reg :: (load (s32) from %stack.0, addrspace 5)
+ ; CHECK-NEXT: $sgpr105 = S_AND_B32 killed renamable $sgpr0, renamable $sgpr0, implicit-def $scc
; CHECK-NEXT: S_NOP 0, implicit $sgpr104, implicit $sgpr105
%0:sreg_64 = COPY $sgpr0_sgpr1
%0.sub0:sreg_64 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index bc8a383a285b2..5e906cc9b8783 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -34,20 +34,16 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3:0x00000000000000FF, $vgpr4_vgpr5_vgpr6_vgpr7:0x00000000000000FF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: SI_SPILL_AV128_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3, %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr0 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr6, 0, killed $vgpr0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: renamable $vgpr0 = V_TRUNC_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
- ; CHECK-NEXT: SI_SPILL_AV32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr7 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, $vgpr7, 0, killed $vgpr3, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr6 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3 = COPY killed renamable $vgpr6_vgpr7
- ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, $vgpr4, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 12, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr3 = nofpexcept V_DIV_FIXUP_F32_e64 0, $vgpr1, 0, $vgpr7, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr2 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7 = SI_SPILL_AV64_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr1, 0, $vgpr4, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $vgpr1 = V_FMA_F32_e64 0, killed $vgpr5, 0, $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SCRATCH_LOAD_DWORDX4_SADDR %stack.0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load (s128), addrspace 5)
; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll b/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
deleted file mode 100644
index 50efc06237d5b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: not llc -mcpu=gfx1100 -mtriple=amdgcn-amd-amdhsa -stress-regalloc=4 -amdgpu-enable-rewrite-partial-reg-uses=0 -filetype=null -verify-machineinstrs %s 2>&1 | FileCheck %s
-
-; CHECK: error: <unknown>:0:0: ran out of registers during register allocation in function 'f'
-; CHECK-NOT: Bad machine code
-
-define <16 x half> @f(i1 %LGV2, <16 x half> %0) {
-BB:
- br i1 %LGV2, label %SW_C3, label %SW_C
-
-SW_C: ; preds = %BB
- %B1 = fmul <16 x half> %0, zeroinitializer
- ret <16 x half> %B1
-
-SW_C3: ; preds = %BB
- ret <16 x half> <half 0xH0000, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison>
-}
>From 66cb85a1aae85888bc26e172e559bf5012fbd94f Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Thu, 8 Jan 2026 17:53:02 +0000
Subject: [PATCH 2/3] test rebase
---
...pressure-mitigation-with-subreg-reload.mir | 24 +++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
index 11e82bacb4dc6..26249c008531f 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
@@ -53,18 +53,18 @@ body: |
; CHECK-NEXT: [[SI_SPILL_V64_RESTORE1:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
; CHECK-NEXT: FLAT_STORE_DWORDX4 [[SI_SPILL_V64_RESTORE1]], [[COPY5]], 0, 0, implicit $exec, implicit $flat_scr
; CHECK-NEXT: SI_RETURN
- %8:vgpr_32 = COPY $vgpr1
- %12:vreg_64 = COPY killed renamable $sgpr0_sgpr1
- %11:vreg_128 = FLAT_LOAD_DWORDX4 %12, 0, 0, implicit $exec, implicit $flat_scr
- undef early-clobber %49.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub0, 42, 0, 0, implicit $exec
- undef %50.sub0:vreg_64 = COPY %49.sub1
- early-clobber %49.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub1, 42, %50, 0, implicit $exec
- early-clobber %15:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub2, 42, 0, 0, implicit $exec
- undef %52.sub0:vreg_64 = COPY %15.sub1
- early-clobber %27:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %11.sub3, 42, %52, 0, implicit $exec
- %49.sub2:vreg_128 = COPY %15.sub0
- %49.sub3:vreg_128 = COPY %27.sub0
- $vgpr31 = COPY %8
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vreg_64 = COPY killed renamable $sgpr0_sgpr1
+ %3:vreg_128 = FLAT_LOAD_DWORDX4 %2, 0, 0, implicit $exec, implicit $flat_scr
+ undef early-clobber %4.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub0, 42, 0, 0, implicit $exec
+ undef %5.sub0:vreg_64 = COPY %4.sub1
+ early-clobber %4.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub1, 42, %5, 0, implicit $exec
+ early-clobber %6:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub2, 42, 0, 0, implicit $exec
+ undef %7.sub0:vreg_64 = COPY %6.sub1
+ early-clobber %8:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 %3.sub3, 42, %7, 0, implicit $exec
+ %4.sub2:vreg_128 = COPY %6.sub0
+ %4.sub3:vreg_128 = COPY %8.sub0
+ $vgpr31 = COPY %1
INLINEASM &"; use v1", 1, 327690, $vgpr1
FLAT_STORE_DWORDX4 %2, %4, 0, 0, implicit $exec, implicit $flat_scr
SI_RETURN
>From 986a1a5c8e81215715beb4e93abb674db6c4a1b6 Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Mon, 12 Jan 2026 13:24:01 +0000
Subject: [PATCH 3/3] lit test update after rebase from main.
---
.../ran-out-of-sgprs-allocation-failure.mir | 185 ++++++++----------
1 file changed, 82 insertions(+), 103 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 8d541f6aadfa2..63b61b6023c53 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -43,102 +43,86 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr15, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.0, addrspace 5)
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr14, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.1, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.3, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.4, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr38_sgpr39 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr48_sgpr49 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr56 = S_MOV_B32 0
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.5, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.5, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr50_sgpr51 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit $exec
- ; CHECK-NEXT: renamable $sgpr100_sgpr101 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr52_sgpr53 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr57 = S_MOV_B32 1083786240
; CHECK-NEXT: SI_SPILL_S1024_SAVE renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, %stack.2, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.2, align 4, addrspace 5)
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr100_sgpr101, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr52_sgpr53, implicit-def dead $scc
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_1024_align2 = COPY [[COPY]]
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
; CHECK-NEXT: S_BRANCH %bb.17
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr41 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr42 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr43 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr44 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr45 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr46 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr47 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr48 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr49 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr50 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr51 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52 = COPY renamable $sgpr56
- ; CHECK-NEXT: renamable $sgpr53 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr56_sgpr57 = COPY renamable $sgpr52_sgpr53
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr56_sgpr57
- ; CHECK-NEXT: renamable $sgpr54 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr48_sgpr49_sgpr50 = COPY renamable $sgpr52_sgpr53_sgpr54
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54 = COPY renamable $sgpr48_sgpr49_sgpr50
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47
- ; CHECK-NEXT: renamable $sgpr55 = COPY killed renamable $sgpr68
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr57 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr58 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr59 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr60 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr61 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr62 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr63 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr64 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr65 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr66 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr68 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = COPY renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr68_sgpr69 = COPY killed renamable $sgpr12_sgpr13
+ ; CHECK-NEXT: renamable $sgpr56 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr57 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr58 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr59 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr60 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr61 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr62 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr63 = COPY renamable $sgpr68
; CHECK-NEXT: renamable $sgpr64 = COPY renamable $sgpr68
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr65 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr66 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr65 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr66 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr68 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr69 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr70 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr71 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr72 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr73 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr74 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr75 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr76 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr77 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr78 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr79 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr80 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr81 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr82 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr83 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr84 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr85 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr86 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr87 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87, implicit $exec
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.11, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
@@ -165,23 +149,22 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.12(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr12_sgpr13, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = S_AND_B64 renamable $sgpr38_sgpr39, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
; CHECK-NEXT: renamable $sgpr54_sgpr55 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: $exec = S_MOV_B64_term killed renamable $sgpr12_sgpr13
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.12, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
; CHECK-NEXT: successors: %bb.7(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
+ ; CHECK-NEXT: dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr48_sgpr49, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7:
; CHECK-NEXT: successors: %bb.8(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $sgpr64_sgpr65 = nofpexcept V_CMP_NLT_F64_e64 0, undef $sgpr4_sgpr5, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
; CHECK-NEXT: renamable $sgpr66_sgpr67 = nofpexcept V_CMP_NLT_F64_e64 0, 4607182418800017408, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
@@ -189,19 +172,19 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.8:
; CHECK-NEXT: successors: %bb.10(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr64_sgpr65, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.10, implicit $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.9:
; CHECK-NEXT: successors: %bb.10(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY killed renamable $sgpr84_sgpr85, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY killed renamable $sgpr12_sgpr13, implicit $exec
; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR undef %18:vgpr_32, [[COPY2]], undef renamable $sgpr4_sgpr5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
- ; CHECK-NEXT: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr14_sgpr15, implicit $exec
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
; CHECK-NEXT: dead renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_1]], implicit $exec
; CHECK-NEXT: renamable $sgpr82 = S_ADD_U32 renamable $sgpr8, 32, implicit-def dead $scc
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
@@ -215,13 +198,11 @@ body: |
; CHECK-NEXT: $sgpr13 = SI_SPILL_S32_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.0, addrspace 5)
; CHECK-NEXT: renamable $sgpr84 = COPY killed renamable $sgpr8
; CHECK-NEXT: renamable $sgpr33 = COPY killed renamable $sgpr16
- ; CHECK-NEXT: renamable $sgpr48_sgpr49 = COPY killed renamable $sgpr14_sgpr15
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: $sgpr8_sgpr9 = COPY renamable $sgpr82_sgpr83
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr8_sgpr9
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = COPY killed renamable $sgpr48_sgpr49
; CHECK-NEXT: renamable $sgpr16 = COPY killed renamable $sgpr33
; CHECK-NEXT: renamable $sgpr4_sgpr5 = COPY killed renamable $sgpr68_sgpr69
; CHECK-NEXT: renamable $sgpr6_sgpr7 = COPY killed renamable $sgpr70_sgpr71
@@ -234,44 +215,42 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.10:
; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.12(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.8, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.12
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.11:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.1, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.17
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.12:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.13(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_MOV_B64_term killed renamable $sgpr54_sgpr55
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.11, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.13:
; CHECK-NEXT: successors: %bb.15(0x40000000), %bb.14(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr50_sgpr51, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.15, implicit $vcc
; CHECK-NEXT: S_BRANCH %bb.14
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.14:
; CHECK-NEXT: successors: %bb.15(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.15:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.16(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr36_sgpr37, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.11, implicit $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.16:
More information about the llvm-branch-commits
mailing list