[llvm] [AMDGPU][DAG] Allow extract_subvector for free register copy (PR #214152)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 03:29:22 PDT 2026
https://github.com/Shoreshen updated https://github.com/llvm/llvm-project/pull/214152
>From ec44c3dc87b8e0dfc0d7d359d82b729b2567f409 Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Wed, 5 Aug 2026 16:14:13 +0800
Subject: [PATCH 1/4] Allow extract_subvector for free register copy
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 +-
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 22 +
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 19 +
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 7 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 185 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 3 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 11 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 6 +
llvm/test/CodeGen/AMDGPU/agpr-csr.ll | 634 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 165 +-
.../AMDGPU/amdgpu-codegenprepare-idiv.ll | 188 +-
.../CodeGen/AMDGPU/any_extend_vector_inreg.ll | 126 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 12 +-
.../AMDGPU/buffer-fat-pointers-memcpy.ll | 75 +-
.../CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll | 18 +-
.../CodeGen/AMDGPU/combine-vload-extract.ll | 16 +-
.../CodeGen/AMDGPU/dag-divergence-atomic.ll | 164 +-
llvm/test/CodeGen/AMDGPU/ds-alignment.ll | 58 +-
.../CodeGen/AMDGPU/extract-subvector-16bit.ll | 28 +-
.../AMDGPU/extract_subvector_to_subreg.ll | 139 +
.../AMDGPU/extract_subvector_vec4_vec3.ll | 11 +-
.../CodeGen/AMDGPU/fence-barrier-latency.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fma.f64.ll | 28 +-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 324 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 64 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 2055 ++++----
llvm/test/CodeGen/AMDGPU/function-returns.ll | 46 +-
.../identical-subrange-spill-infloop.ll | 14 +-
.../test/CodeGen/AMDGPU/image-load-d16-tfe.ll | 4 +-
.../CodeGen/AMDGPU/indirect-addressing-si.ll | 1571 +++---
.../AMDGPU/infer-addrspace-flat-atomic.ll | 18 +-
llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll | 4 +-
.../CodeGen/AMDGPU/insert_vector_dynelt.ll | 1960 +++----
llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll | 524 +-
.../CodeGen/AMDGPU/invalid-cast-load-i1.ll | 3 +-
llvm/test/CodeGen/AMDGPU/kernel-args.ll | 419 +-
.../AMDGPU/kernel-argument-dag-lowering.ll | 22 +-
...llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll | 24 +-
.../CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll | 272 +-
.../llvm.amdgcn.image.sample.d16.dim.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll | 115 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 36 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll | 60 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 368 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 1470 ++----
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll | 41 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 1832 ++++---
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 302 +-
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 640 +--
.../AMDGPU/llvm.amdgcn.permlane64.ptr.ll | 19 +-
...vm.amdgcn.sched.group.barrier.iterative.ll | 102 +-
.../AMDGPU/llvm.amdgcn.sched.group.barrier.ll | 272 +-
.../AMDGPU/llvm.amdgcn.smfmac.gfx950.ll | 120 +-
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 746 ++-
llvm/test/CodeGen/AMDGPU/load-constant-i32.ll | 196 +-
llvm/test/CodeGen/AMDGPU/load-constant-i64.ll | 415 +-
llvm/test/CodeGen/AMDGPU/load-global-f32.ll | 99 +-
llvm/test/CodeGen/AMDGPU/load-global-i16.ll | 274 +-
llvm/test/CodeGen/AMDGPU/load-global-i32.ll | 299 +-
llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 1572 +++---
...uffer-fat-pointers-nontemporal-metadata.ll | 26 +-
.../CodeGen/AMDGPU/memintrinsic-unroll.ll | 4508 +++++++++--------
.../AMDGPU/memset-param-combinations.ll | 120 +-
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll | 320 +-
.../AMDGPU/mfma-no-register-aliasing.ll | 72 +-
.../AMDGPU/no-folding-imm-to-inst-with-fi.ll | 75 +-
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 248 +-
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 180 +-
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 164 +-
llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll | 40 +-
llvm/test/CodeGen/AMDGPU/preload-kernargs.ll | 38 +-
.../AMDGPU/ptradd-sdag-optimizations.ll | 29 +-
.../AMDGPU/rewrite-mfma-single-exit-copy.ll | 769 ++-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 11 +-
llvm/test/CodeGen/AMDGPU/scratch-simple.ll | 1953 +++----
llvm/test/CodeGen/AMDGPU/shl.v2i64.ll | 163 +-
.../AMDGPU/shufflevector-physreg-copy.ll | 23 +-
.../AMDGPU/shufflevector.v2f32.v4f32.ll | 32 +-
.../AMDGPU/shufflevector.v2f32.v8f32.ll | 44 +-
.../AMDGPU/shufflevector.v2i32.v4i32.ll | 32 +-
.../AMDGPU/shufflevector.v2i32.v8i32.ll | 20 +-
.../CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll | 32 +-
.../CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll | 20 +-
.../AMDGPU/shufflevector.v4f32.v2f32.ll | 223 +-
.../AMDGPU/shufflevector.v4f32.v3f32.ll | 169 +-
.../AMDGPU/shufflevector.v4f32.v4f32.ll | 668 ++-
.../AMDGPU/shufflevector.v4i32.v2i32.ll | 223 +-
.../AMDGPU/shufflevector.v4i32.v3i32.ll | 169 +-
.../AMDGPU/shufflevector.v4i32.v4i32.ll | 668 ++-
.../CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll | 223 +-
.../CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll | 169 +-
.../CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll | 668 ++-
.../AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll | 201 +-
llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll | 6 +-
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 2936 +++++------
llvm/test/CodeGen/AMDGPU/spill-vgpr.ll | 2 +-
.../AMDGPU/splitkit-getsubrangeformask.ll | 34 +-
llvm/test/CodeGen/AMDGPU/sra.ll | 30 +-
llvm/test/CodeGen/AMDGPU/srem.ll | 1119 ++--
llvm/test/CodeGen/AMDGPU/store-local.128.ll | 20 +-
llvm/test/CodeGen/AMDGPU/store-local.96.ll | 86 +-
llvm/test/CodeGen/AMDGPU/store-to-constant.ll | 6 +-
.../AMDGPU/undef-handling-crash-in-ra.ll | 22 +-
.../CodeGen/AMDGPU/vector-reduce-fmaximum.ll | 300 +-
.../CodeGen/AMDGPU/vector-reduce-fminimum.ll | 300 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 42 +-
.../AMDGPU/vgpr-mark-last-scratch-load.ll | 38 +-
.../test/CodeGen/AMDGPU/vni8-across-blocks.ll | 132 +-
...wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 14 +-
110 files changed, 17570 insertions(+), 18050 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/extract_subvector_to_subreg.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 418ef38daac29..5261b3e9f40fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -28268,8 +28268,15 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
V.getOperand(0), NewIndex);
return DAG.getBitcast(NVT, NewExtract);
}
+ // The action for EXTRACT_VECTOR_ELT is registered on the type of the
+ // vector being read, so once operations have been legalized SrcVT is
+ // what decides whether the replacement is really legal. Reading from
+ // an illegal source vector would be promoted straight back into the
+ // extract_subvector we started from.
if (NewExtEC.isScalar() &&
- TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) {
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT) &&
+ (!LegalOperations ||
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
SDValue NewExtract =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a6fa716fd1598..9c18354937801 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -679,6 +679,25 @@ void AMDGPUDAGToDAGISel::SelectVectorShuffle(SDNode *N) {
CurDAG->SelectNodeTo(N, TargetOpcode::REG_SEQUENCE, VT, Ops);
}
+// Extracts that LowerEXTRACT_SUBVECTOR left alone are plain subregister
+// references.
+void AMDGPUDAGToDAGISel::SelectExtractSubvector(SDNode *N) {
+ const auto &TLI =
+ static_cast<const AMDGPUTargetLowering &>(*getTargetLowering());
+
+ EVT VT = N->getValueType(0);
+ SDValue Src = N->getOperand(0);
+ unsigned SubReg = TLI.getExtractSubvectorSubReg(VT, Src.getValueType(),
+ N->getConstantOperandVal(1));
+ if (SubReg == AMDGPU::NoSubRegister) {
+ SelectCode(N);
+ return;
+ }
+
+ ReplaceNode(
+ N, CurDAG->getTargetExtractSubreg(SubReg, SDLoc(N), VT, Src).getNode());
+}
+
void AMDGPUDAGToDAGISel::Select(SDNode *N) {
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
@@ -757,6 +776,9 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
case ISD::VECTOR_SHUFFLE:
SelectVectorShuffle(N);
return;
+ case ISD::EXTRACT_SUBVECTOR:
+ SelectExtractSubvector(N);
+ return;
case ISD::BUILD_PAIR: {
SDValue RC, SubReg0, SubReg1;
SDLoc DL(N);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index 95f85a6151375..2a32f12ac84e1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -74,6 +74,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
protected:
void SelectBuildVector(SDNode *N, unsigned RegClassID);
void SelectVectorShuffle(SDNode *N);
+ void SelectExtractSubvector(SDNode *N);
bool isSDWAOperand(const SDNode *N) const;
private:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 962988ff97e39..0c34aa399d861 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -832,6 +832,11 @@ bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
}
+unsigned AMDGPUTargetLowering::getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
+ return AMDGPU::NoSubRegister;
+}
+
bool AMDGPUTargetLowering::shouldReduceLoadWidth(
SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
std::optional<unsigned> ByteOffset) const {
@@ -839,6 +844,13 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(
if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT, ByteOffset))
return false;
+ // WidenOrSplitVectorLoad legalizes a 3 element vector load by loading 4
+ // elements and extracting the low 3 back out. Narrowing a load down to 3
+ // elements would just be undone by that, and the two would keep undoing each
+ // other.
+ if (NewVT.isVector() && NewVT.getVectorNumElements() == 3)
+ return false;
+
unsigned NewSize = NewVT.getStoreSizeInBits();
// If we are reducing to a 32-bit load or a smaller multi-dword load,
@@ -1642,6 +1654,13 @@ SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
EVT VT = Op.getValueType();
EVT SrcVT = Op.getOperand(0).getValueType();
+ // A free extract is a sub register reference, which isel turns into an
+ // EXTRACT_SUBREG. This has to be checked before the 16-bit case below:
+ // rewriting a free extract into a build_vector only gets folded straight back
+ // into the extract by the combiner, and legalizing that rewrites it again.
+ if (getExtractSubvectorSubReg(VT, SrcVT, Start) != AMDGPU::NoSubRegister)
+ return Op;
+
if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
unsigned NumElt = VT.getVectorNumElements();
unsigned NumSrcElt = SrcVT.getVectorNumElements();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index f24f33e7dcb70..c25276580c4ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -233,6 +233,13 @@ class AMDGPUTargetLowering : public TargetLowering {
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtType, EVT ExtVT,
std::optional<unsigned> ByteOffset) const override;
+ /// If extracting a subvector of type \p ResVT starting at element \p Index
+ /// out of a vector of type \p SrcVT is a plain sub register reference,
+ /// return the sub register index that describes it. Otherwise return
+ /// AMDGPU::NoSubRegister.
+ virtual unsigned getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const;
+
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG,
const MachineMemOperand &MMO) const final;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9159e425dfffd..5ff4720dbc6ee 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1108,6 +1108,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::SIGN_EXTEND_INREG,
ISD::ANY_EXTEND,
ISD::EXTRACT_VECTOR_ELT,
+ ISD::EXTRACT_SUBVECTOR,
ISD::INSERT_VECTOR_ELT,
ISD::FCOPYSIGN});
@@ -2407,11 +2408,61 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
return true;
}
+unsigned SITargetLowering::getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
+ if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
+ return AMDGPU::NoSubRegister;
+
+ // Both types have to be assigned a register class, otherwise the result
+ // cannot be named as a sub register of the source register.
+ if (!ResVT.isSimple() || !SrcVT.isSimple() || !isTypeLegal(ResVT) ||
+ !isTypeLegal(SrcVT))
+ return AMDGPU::NoSubRegister;
+
+ // A single register wide result is deliberately not described as a sub
+ // register reference: reading one register out of a tuple costs the same
+ // whether it is spelled as an extract_subvector or as an extract_vector_elt,
+ // and keeping the element form lets the combiner see what each element holds.
+ unsigned SizeInBits = ResVT.getSizeInBits();
+ if (SizeInBits <= 32)
+ return AMDGPU::NoSubRegister;
+
+ // Registers are 32 bits wide, so a piece that does not start and end on a
+ // register boundary has to be assembled by real instructions.
+ unsigned OffsetInBits = Index * SrcVT.getScalarSizeInBits();
+ if (SizeInBits % 32 != 0 || OffsetInBits % 32 != 0)
+ return AMDGPU::NoSubRegister;
+
+ unsigned SubReg = SIRegisterInfo::getSubRegFromChannelOrNone(
+ OffsetInBits / 32, SizeInBits / 32);
+ if (SubReg == AMDGPU::NoSubRegister)
+ return AMDGPU::NoSubRegister;
+
+ // Divergence is not known here, so require the sub register to be usable
+ // with the source register class regardless of which register file the
+ // value ends up in.
+ const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
+ for (bool IsDivergent : {false, true}) {
+ const TargetRegisterClass *SrcRC =
+ getRegClassFor(SrcVT.getSimpleVT(), IsDivergent);
+ if (!TRI->getSubClassWithSubReg(SrcRC, SubReg))
+ return AMDGPU::NoSubRegister;
+ }
+
+ return SubReg;
+}
+
bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
unsigned Index) const {
if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
return false;
+ // A plain sub register reference is free, and isel turns it into an
+ // EXTRACT_SUBREG. Everything else still goes through the elementwise
+ // lowering in AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR.
+ if (getExtractSubvectorSubReg(ResVT, SrcVT, Index) != AMDGPU::NoSubRegister)
+ return true;
+
// TODO: Add more cases that are cheap.
return Index == 0;
}
@@ -16888,6 +16939,33 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
unsigned VecSize = VecVT.getSizeInBits();
unsigned VecEltSize = VecEltVT.getSizeInBits();
+ // extract_vector_elt (extract_subvector X, I), J
+ // -> extract_vector_elt X, I + J
+ //
+ // A free extract is a subregister reference, so reading an element out of it
+ // reads the same register as reading that element out of the vector it came
+ // from. Going straight to the source lets the subvector stop being
+ // referenced, so it no longer needs a register of its own.
+ if (Vec.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+ SDValue Src = Vec.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ if (auto *EltIdx = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
+ // The action for EXTRACT_VECTOR_ELT is registered on the type of the
+ // vector being read, so the source is what decides whether reading the
+ // element straight out of it is really legal.
+ if (getExtractSubvectorSubReg(VecVT, SrcVT,
+ Vec.getConstantOperandVal(1)) !=
+ AMDGPU::NoSubRegister &&
+ isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT)) {
+ SDValue NewExt = DAG.getExtractVectorElt(SDLoc(N), ResVT, Src,
+ Vec.getConstantOperandVal(1) +
+ EltIdx->getZExtValue());
+ NewExt->setIROrder(Src->getIROrder());
+ return NewExt;
+ }
+ }
+ }
+
if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
allUsesHaveSourceMods(N)) {
SDLoc SL(N);
@@ -17038,6 +17116,38 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
return SDValue();
}
+// extract_subvector (extract_subvector X, I), J -> extract_subvector X, I + J
+//
+// The generic combiner only does this when the inner extract has a single use,
+// because in general each extract it leaves behind is an instruction. A free
+// extract is just a subregister reference, so here it is worth repeating for
+// every use: the intermediate vector stops being referenced and no longer needs
+// a register of its own, which matters most for the wide tuples this reads
+// from.
+SDValue
+SITargetLowering::performExtractSubvectorCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SDValue Src = N->getOperand(0);
+ if (Src.getOpcode() != ISD::EXTRACT_SUBVECTOR || Src.hasOneUse())
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ SDValue Outer = Src.getOperand(0);
+ unsigned Start = Src.getConstantOperandVal(1) + N->getConstantOperandVal(1);
+
+ // The combined index is relative to a wider vector than the one the inner
+ // extract indexed into, and an extract_subvector index has to stay a multiple
+ // of the number of elements it produces.
+ if (Start % VT.getVectorMinNumElements() != 0)
+ return SDValue();
+
+ if (getExtractSubvectorSubReg(VT, Outer.getValueType(), Start) ==
+ AMDGPU::NoSubRegister)
+ return SDValue();
+
+ return DCI.DAG.getExtractSubvector(SDLoc(N), VT, Outer, Start);
+}
+
SDValue
SITargetLowering::performInsertVectorEltCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -19026,6 +19136,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
}
case ISD::EXTRACT_VECTOR_ELT:
return performExtractVectorEltCombine(N, DCI);
+ case ISD::EXTRACT_SUBVECTOR:
+ return performExtractSubvectorCombine(N, DCI);
case ISD::INSERT_VECTOR_ELT:
return performInsertVectorEltCombine(N, DCI);
case ISD::FP_ROUND:
@@ -19048,22 +19160,45 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
}
-/// Helper function for adjustWritemask
-static unsigned SubIdx2Lane(unsigned Idx) {
- switch (Idx) {
- default:
- return ~0u;
- case AMDGPU::sub0:
- return 0;
- case AMDGPU::sub1:
- return 1;
- case AMDGPU::sub2:
- return 2;
- case AMDGPU::sub3:
- return 3;
- case AMDGPU::sub4:
- return 4; // Possible with TFE/LWE
+/// Helper function for adjustWritemask. Collect the EXTRACT_SUBREG users of
+/// \p Node that read a single 32-bit register into \p Lanes, together with the
+/// lane they read. \p LaneOffset is the lane \p Node itself starts at.
+///
+/// EXTRACT_SUBREGs that name a wider piece are looked through, since they only
+/// rename a range of lanes; the reads of the individual registers are what
+/// decides which channels are live.
+///
+/// \returns false if any use cannot be described as a lane read.
+static bool
+collectMIMGLaneUsers(SDNode *Node, unsigned LaneOffset,
+ const SIRegisterInfo &TRI,
+ SmallVectorImpl<std::pair<SDNode *, unsigned>> &Lanes) {
+ for (SDUse &Use : Node->uses()) {
+ // Don't look at users of the chain.
+ if (Use.getResNo() != 0)
+ continue;
+
+ SDNode *User = Use.getUser();
+ if (!User->isMachineOpcode() ||
+ User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
+ return false;
+
+ unsigned SubReg = User->getConstantOperandVal(1);
+ unsigned Offset = TRI.getSubRegIdxOffset(SubReg);
+ unsigned Size = TRI.getSubRegIdxSize(SubReg);
+ if (Offset % 32 != 0 || Size % 32 != 0)
+ return false;
+
+ if (Size == 32) {
+ Lanes.emplace_back(User, LaneOffset + Offset / 32);
+ continue;
+ }
+
+ if (!collectMIMGLaneUsers(User, LaneOffset + Offset / 32, TRI, Lanes))
+ return false;
}
+
+ return true;
}
/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
@@ -19101,25 +19236,17 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
}
// Try to figure out the used register components
- for (SDUse &Use : Node->uses()) {
-
- // Don't look at users of the chain.
- if (Use.getResNo() != 0)
- continue;
-
- SDNode *User = Use.getUser();
-
- // Abort if we can't understand the usage
- if (!User->isMachineOpcode() ||
- User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
- return Node;
+ SmallVector<std::pair<SDNode *, unsigned>, 8> LaneUses;
+ if (!collectMIMGLaneUsers(Node, 0, *Subtarget->getRegisterInfo(), LaneUses))
+ return Node;
+ for (auto [User, UserLane] : LaneUses) {
// Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
// Note that subregs are packed, i.e. Lane==0 is the first bit set
// in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
// set, etc.
- Lane = SubIdx2Lane(User->getConstantOperandVal(1));
- if (Lane == ~0u)
+ Lane = UserLane;
+ if (Lane >= std::size(Users))
return Node;
// Check if the use is for the TFE/LWE generated result at VGPRn+1.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f358f59311c79..8413d74dc823a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -234,6 +234,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performExtractSubvectorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFPRoundCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
@@ -405,6 +406,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
Type *Ty) const override;
+ unsigned getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const override;
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
unsigned Index) const override;
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index a3d6509628d50..8462bdd69303f 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -578,6 +578,17 @@ unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
return SubRegFromChannelTable[NumRegIndex - 1][Channel];
}
+unsigned SIRegisterInfo::getSubRegFromChannelOrNone(unsigned Channel,
+ unsigned NumRegs) {
+ if (NumRegs >= SubRegFromChannelTableWidthMap.size())
+ return AMDGPU::NoSubRegister;
+ unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs];
+ if (NumRegIndex == 0 ||
+ Channel >= SubRegFromChannelTable[NumRegIndex - 1].size())
+ return AMDGPU::NoSubRegister;
+ return SubRegFromChannelTable[NumRegIndex - 1][Channel];
+}
+
bool SIRegisterInfo::isCFISavedRegsSpillEnabled() const {
return EnableSpillCFISavedRegs;
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index e464c9334ffea..4502bfc7d4509 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -76,6 +76,12 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
/// (e.g. getSubRegFromChannel(0) -> AMDGPU::sub0)
static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs = 1);
+ /// Same as getSubRegFromChannel, but returns AMDGPU::NoSubRegister instead of
+ /// asserting when \p NumRegs registers starting at \p Channel are not
+ /// describable by a single sub register index.
+ static unsigned getSubRegFromChannelOrNone(unsigned Channel,
+ unsigned NumRegs = 1);
+
bool spillSGPRToVGPR() const {
return SpillSGPRToVGPR;
}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
index 016ac4eeb8867..78549067ff5e7 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
@@ -180,55 +180,63 @@ define amdgpu_kernel void @test_call_empty() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v32, a28
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -313,57 +321,65 @@ define amdgpu_kernel void @test_call_areg4() #0 {
; GFX908-NEXT: s_mov_b64 s[2:3], s[22:23]
; GFX908-NEXT: s_mov_b32 s32, 0
; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def a[0:31]
+; GFX908-NEXT: ; def a[4:35]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v32, a28
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a32
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a33
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a34
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a35
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -448,57 +464,65 @@ define amdgpu_kernel void @test_call_areg32() #0 {
; GFX908-NEXT: s_mov_b64 s[2:3], s[22:23]
; GFX908-NEXT: s_mov_b32 s32, 0
; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def a[0:31]
+; GFX908-NEXT: ; def a[32:63]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v32, a28
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a60
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a61
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a62
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a63
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a56
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a57
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a58
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a59
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a52
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a53
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a54
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a55
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a48
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a49
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a50
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a51
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a44
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a45
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a46
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a47
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a40
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a41
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a42
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a43
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a36
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a37
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a38
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a39
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a32
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a33
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a34
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a35
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -585,55 +609,63 @@ define amdgpu_kernel void @test_call_areg64() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v32, a28
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -718,57 +750,65 @@ define amdgpu_kernel void @test_call_areg31_63() #0 {
; GFX908-NEXT: s_mov_b64 s[2:3], s[22:23]
; GFX908-NEXT: s_mov_b32 s32, 0
; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def a[0:31]
+; GFX908-NEXT: ; def a[64:95]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v32, a28
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a92
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a93
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a94
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a95
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a88
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a89
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a90
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a91
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a84
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a85
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a86
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a87
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a80
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a81
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a82
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a83
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a76
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a77
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a78
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a79
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a72
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a73
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a74
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a75
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a68
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a69
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a70
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a71
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a64
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a65
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a66
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a67
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -846,64 +886,96 @@ define amdgpu_kernel void @test_call_unknown() #0 {
; GFX908-NEXT: v_lshlrev_b32_e32 v1, 10, v1
; GFX908-NEXT: s_mov_b32 s14, s10
; GFX908-NEXT: s_mov_b64 s[10:11], s[6:7]
+; GFX908-NEXT: ;;#ASMSTART
+; GFX908-NEXT: ; def a[0:31]
+; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: v_or3_b32 v31, v0, v1, v2
; GFX908-NEXT: s_mov_b64 s[4:5], s[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX908-NEXT: s_mov_b64 s[0:1], s[36:37]
; GFX908-NEXT: s_mov_b64 s[2:3], s[38:39]
; GFX908-NEXT: s_mov_b32 s32, 0
-; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def a[0:31]
-; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v43, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v42, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v41, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v40, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v47, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v46, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v45, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v44, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v59, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v58, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v57, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v56, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v63, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v62, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v61, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v60, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v75, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v74, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v73, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v72, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v79, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v78, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v77, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v76, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v91, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v90, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v89, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v88, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v95, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v94, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v93, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v92, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v95, a0 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v94, a1 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v93, a2 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v92, a3 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v91, a4 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v90, a5 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v89, a6 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v88, a7 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v79, a8 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v78, a9 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v77, a10 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v76, a11 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v75, a12 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v74, a13 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v73, a14 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v72, a15 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v63, a16 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v62, a17 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v61, a18 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v60, a19 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v59, a20 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v58, a21 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v57, a22 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v56, a23 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v47, a24 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v46, a25 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v45, a26 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v44, a27 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v43, a28 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v42, a29 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v41, a30 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v40, a31 ; Reload Reuse
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[92:95], off
+; GFX908-NEXT: v_mov_b32_e32 v4, v95
+; GFX908-NEXT: v_mov_b32_e32 v5, v94
+; GFX908-NEXT: v_mov_b32_e32 v6, v93
+; GFX908-NEXT: v_mov_b32_e32 v7, v92
+; GFX908-NEXT: v_mov_b32_e32 v8, v91
+; GFX908-NEXT: v_mov_b32_e32 v9, v90
+; GFX908-NEXT: v_mov_b32_e32 v10, v89
+; GFX908-NEXT: v_mov_b32_e32 v11, v88
+; GFX908-NEXT: v_mov_b32_e32 v12, v79
+; GFX908-NEXT: v_mov_b32_e32 v13, v78
+; GFX908-NEXT: v_mov_b32_e32 v14, v77
+; GFX908-NEXT: v_mov_b32_e32 v15, v76
+; GFX908-NEXT: v_mov_b32_e32 v16, v75
+; GFX908-NEXT: v_mov_b32_e32 v17, v74
+; GFX908-NEXT: v_mov_b32_e32 v18, v73
+; GFX908-NEXT: v_mov_b32_e32 v19, v72
+; GFX908-NEXT: v_mov_b32_e32 v20, v63
+; GFX908-NEXT: v_mov_b32_e32 v21, v62
+; GFX908-NEXT: v_mov_b32_e32 v22, v61
+; GFX908-NEXT: v_mov_b32_e32 v23, v60
+; GFX908-NEXT: v_mov_b32_e32 v24, v59
+; GFX908-NEXT: v_mov_b32_e32 v25, v58
+; GFX908-NEXT: v_mov_b32_e32 v26, v57
+; GFX908-NEXT: v_mov_b32_e32 v27, v56
+; GFX908-NEXT: v_mov_b32_e32 v28, v47
+; GFX908-NEXT: v_mov_b32_e32 v29, v46
+; GFX908-NEXT: v_mov_b32_e32 v30, v45
+; GFX908-NEXT: v_mov_b32_e32 v31, v44
+; GFX908-NEXT: v_mov_b32_e32 v32, v43
+; GFX908-NEXT: v_mov_b32_e32 v33, v42
+; GFX908-NEXT: v_mov_b32_e32 v34, v41
+; GFX908-NEXT: v_mov_b32_e32 v35, v40
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[88:91], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[28:31], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[76:79], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[72:75], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[60:63], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[56:59], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[44:47], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[40:43], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 005d3672f784b..0cb657aef2ab3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -9869,6 +9869,7 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v16, v1
; GFX9-NEXT: v_mov_b32_e32 v15, v0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; GFX9-NEXT: ; kill: def $vgpr14 killed $sgpr4 killed $exec
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: ; implicit-def: $vgpr3
@@ -9922,6 +9923,7 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v2 :: v_dual_mov_b32 v14, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; kill: def $vgpr12 killed $sgpr0 killed $exec
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
@@ -9978,6 +9980,7 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v16, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, v0
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT: ; kill: def $vgpr14 killed $sgpr0 killed $exec
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr1
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr2
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr3
@@ -11277,10 +11280,11 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v6i16_to_v12i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v8, v2
-; GFX9-NEXT: v_mov_b32_e32 v14, v1
-; GFX9-NEXT: v_mov_b32_e32 v13, v0
+; GFX9-NEXT: v_mov_b32_e32 v13, v2
+; GFX9-NEXT: v_mov_b32_e32 v16, v1
+; GFX9-NEXT: v_mov_b32_e32 v15, v0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; GFX9-NEXT: ; kill: def $vgpr14 killed $sgpr4 killed $exec
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: ; implicit-def: $vgpr3
@@ -11294,43 +11298,45 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB46_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX9-NEXT: .LBB46_2: ; %Flow
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB46_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
-; GFX9-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT: v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT: v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX9-NEXT: .LBB46_4: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v13
-; GFX9-NEXT: v_mov_b32_e32 v4, v14
+; GFX9-NEXT: v_mov_b32_e32 v0, v15
+; GFX9-NEXT: v_mov_b32_e32 v4, v16
+; GFX9-NEXT: v_mov_b32_e32 v8, v13
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v6i16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v2 :: v_dual_mov_b32 v14, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; kill: def $vgpr12 killed $sgpr0 killed $exec
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
@@ -11339,51 +11345,54 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB46_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v14, v1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, v0
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT: ; kill: def $vgpr14 killed $sgpr0 killed $exec
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr1
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr2
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr3
@@ -11398,36 +11407,38 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB46_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX11-FAKE16-NEXT: .LBB46_2: ; %Flow
; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
-; GFX11-FAKE16-NEXT: v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
; GFX11-FAKE16-NEXT: v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX11-FAKE16-NEXT: .LBB46_4: ; %end
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v13
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v14
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v15
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v13
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index fb78590bd110c..c39752134ecd9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -2474,47 +2474,47 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s8, s2, 16
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT: v_rcp_f32_e32 v1, v1
-; GFX9-NEXT: s_lshr_b32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_rcp_f32_e32 v1, v3
+; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s0
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s5
; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_u32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[6:7]
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: global_store_dwordx2 v5, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2661,50 +2661,50 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
-; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX9-NEXT: s_and_b32 s8, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX9-NEXT: s_and_b32 s9, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s9
+; GFX9-NEXT: s_lshr_b32 s8, s2, 16
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
+; GFX9-NEXT: s_lshr_b32 s9, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v4
-; GFX9-NEXT: s_and_b32 s9, s3, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s9
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: s_and_b32 s10, s3, 0xffff
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
-; GFX9-NEXT: s_and_b32 s8, s1, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s8
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s10
+; GFX9-NEXT: s_and_b32 s10, s1, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s10
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s4
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s3
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s1
; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s5
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
-; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s3
+; GFX9-NEXT: v_mul_lo_u32 v4, v4, s4
+; GFX9-NEXT: v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT: v_sub_u32_sdwa v5, s9, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_e32 v1, s1, v3
+; GFX9-NEXT: v_sub_u32_sdwa v3, s5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
@@ -3029,13 +3029,13 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sext_i32_i16 s5, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: s_ashr_i32 s5, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
; GFX9-NEXT: s_sext_i32_i16 s4, s0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_ashr_i32 s8, s0, 16
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
@@ -3043,15 +3043,15 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: s_sext_i32_i16 s9, s3
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
-; GFX9-NEXT: s_sext_i32_i16 s8, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
-; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: s_sext_i32_i16 s4, s1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s4
+; GFX9-NEXT: s_ashr_i32 s4, s3, 16
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s4
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s9, s1, 16
; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s9
; GFX9-NEXT: v_rcp_f32_e32 v5, v5
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
@@ -3063,14 +3063,14 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
-; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s5
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s3
+; GFX9-NEXT: v_mul_lo_u32 v4, v4, s4
+; GFX9-NEXT: v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT: v_sub_u32_sdwa v5, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_e32 v1, s1, v3
+; GFX9-NEXT: v_sub_u32_sdwa v3, s9, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
@@ -3443,15 +3443,15 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_lshr_b32 s5, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s5
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
@@ -3588,15 +3588,15 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: s_lshr_b32 s6, s2, 16
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_lshr_b32 s8, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s7
; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: v_rcp_f32_e32 v2, v2
@@ -3616,12 +3616,12 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s6
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
-; GFX9-NEXT: v_sub_u32_sdwa v2, s8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
+; GFX9-NEXT: v_sub_u32_sdwa v2, s7, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
index 7b6197e3fcb70..c97cc4706edab 100644
--- a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
@@ -57,92 +57,92 @@ define amdgpu_kernel void @any_extend_vector_inreg_v16i8_to_v4i32(ptr addrspace(
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s0, s[8:9], 0x20
-; GFX8-NEXT: s_lshr_b32 s6, s5, 24
-; GFX8-NEXT: s_lshr_b32 s8, s2, 24
+; GFX8-NEXT: s_lshr_b32 s4, s2, 24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshl_b64 s[2:3], s[0:1], 8
-; GFX8-NEXT: s_add_u32 s4, s10, 13
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 15
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 14
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 8
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 11
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 10
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 4
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 6
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 1
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
-; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 3
+; GFX8-NEXT: s_add_u32 s8, s10, 13
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 15
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 14
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 8
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 11
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 10
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 4
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 6
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 1
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 3
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 9
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: s_add_u32 s8, s10, 9
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: s_add_u32 s6, s10, 2
; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_add_u32 s4, s10, 2
-; GFX8-NEXT: s_addc_u32 s5, s11, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s7
+; GFX8-NEXT: s_addc_u32 s7, s11, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_add_u32 s0, s10, 5
; GFX8-NEXT: flat_store_byte v[0:1], v2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v2, s1
; GFX8-NEXT: s_addc_u32 s1, s11, 0
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_add_u32 s0, s10, 12
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: s_add_u32 s0, s10, 7
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: s_addc_u32 s1, s11, 0
+; GFX8-NEXT: s_lshr_b32 s2, s5, 24
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_add_u32 s0, s10, 7
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: s_add_u32 s0, s10, 12
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: s_addc_u32 s1, s11, 0
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index deb8f6e415b70..2c56a38090e97 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -2210,16 +2210,16 @@ define void @test_load_store_v16bf16(ptr addrspace(1) %in, ptr addrspace(1) %out
; GFX8-LABEL: test_load_store_v16bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GFX8-NEXT: flat_store_dwordx4 v[2:3], v[8:11]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index 70106cf0838cd..67316d8a8c053 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -235,21 +235,26 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-LABEL: memcpy_known:
; SDAG-GFX942: ; %bb.0:
; SDAG-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; SDAG-GFX942-NEXT: s_load_dword s17, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
-; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_load_dword s12, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s5, s16
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
-; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
+; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
+; SDAG-GFX942-NEXT: s_mov_b32 s4, s3
+; SDAG-GFX942-NEXT: s_or_b64 s[6:7], s[4:5], s[16:17]
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX942-NEXT: s_mov_b32 s17, s12
+; SDAG-GFX942-NEXT: s_mov_b32 s2, s11
+; SDAG-GFX942-NEXT: s_mov_b32 s3, s16
+; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
+; SDAG-GFX942-NEXT: s_or_b64 s[14:15], s[2:3], s[16:17]
; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
-; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
; SDAG-GFX942-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -266,10 +271,10 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x2000
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -756,21 +761,26 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-LABEL: memcpy_known_medium:
; SDAG-GFX942: ; %bb.0:
; SDAG-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; SDAG-GFX942-NEXT: s_load_dword s17, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
-; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_load_dword s12, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s5, s16
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
-; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
+; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
+; SDAG-GFX942-NEXT: s_mov_b32 s4, s3
+; SDAG-GFX942-NEXT: s_or_b64 s[6:7], s[4:5], s[16:17]
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX942-NEXT: s_mov_b32 s17, s12
+; SDAG-GFX942-NEXT: s_mov_b32 s2, s11
+; SDAG-GFX942-NEXT: s_mov_b32 s3, s16
+; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
+; SDAG-GFX942-NEXT: s_or_b64 s[14:15], s[2:3], s[16:17]
; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
-; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
; SDAG-GFX942-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -787,10 +797,10 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x100
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -1106,19 +1116,24 @@ define amdgpu_kernel void @memcpy_known_small(ptr addrspace(7) %src, ptr addrspa
; SDAG-GFX942-LABEL: memcpy_known_small:
; SDAG-GFX942: ; %bb.0:
; SDAG-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; SDAG-GFX942-NEXT: s_load_dword s11, s[4:5], 0x34
+; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; SDAG-GFX942-NEXT: s_mov_b32 s6, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s11, s6
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX942-NEXT: s_mov_b32 s10, s3
; SDAG-GFX942-NEXT: s_mov_b32 s8, s1
+; SDAG-GFX942-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
; SDAG-GFX942-NEXT: s_mov_b32 s9, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s10, s3
; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s0
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[8:11], 0 offen
-; SDAG-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
+; SDAG-GFX942-NEXT: s_mov_b32 s5, s6
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX942-NEXT: s_mov_b32 s4, s3
+; SDAG-GFX942-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
; SDAG-GFX942-NEXT: v_mov_b32_e32 v1, s0
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(0)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v1, s[4:7], 0 offen
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
index 61ff755f99182..b32c9f8da6d7b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
@@ -26,7 +26,7 @@ define amdgpu_kernel void @buffer_ptr_vector_ops(ptr addrspace(1) %somewhere) {
; SDAG-LABEL: buffer_ptr_vector_ops:
; SDAG: ; %bb.0: ; %main_body
; SDAG-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; SDAG-NEXT: v_mov_b32_e32 v8, 0
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -34,13 +34,17 @@ define amdgpu_kernel void @buffer_ptr_vector_ops(ptr addrspace(1) %somewhere) {
; SDAG-NEXT: v_mov_b32_e32 v1, s1
; SDAG-NEXT: v_mov_b32_e32 v2, s2
; SDAG-NEXT: v_mov_b32_e32 v3, s3
-; SDAG-NEXT: v_mov_b32_e32 v4, s4
-; SDAG-NEXT: v_mov_b32_e32 v5, s5
-; SDAG-NEXT: v_mov_b32_e32 v6, s6
-; SDAG-NEXT: v_mov_b32_e32 v7, s7
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v8, s4
+; SDAG-NEXT: v_mov_b32_e32 v9, s5
+; SDAG-NEXT: v_mov_b32_e32 v10, s6
+; SDAG-NEXT: v_mov_b32_e32 v11, s7
; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] offset:48
-; SDAG-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:32
+; SDAG-NEXT: global_store_dwordx4 v12, v[4:7], s[8:9] offset:48
+; SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[8:9] offset:32
; SDAG-NEXT: s_endpgm
main_body:
%buffers = load <2 x ptr addrspace(8)>, ptr addrspace(1) %somewhere
diff --git a/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll b/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
index a2bf889fe3297..31b3c89917aaa 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
@@ -6,11 +6,9 @@ define amdgpu_kernel void @vectorLoadCombine(ptr %in, ptr %out) {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GCN-NEXT: flat_load_dword v2, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -39,14 +37,12 @@ define amdgpu_kernel void @vectorLoadShuffle(ptr %in, ptr %out) {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: flat_load_dword v2, v[0:1]
+; GCN-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GCN-NEXT: flat_load_dword v0, v[0:1]
; GCN-NEXT: s_mov_b32 s0, 0x7050604
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_perm_b32 v2, v2, v2, s0
+; GCN-NEXT: v_perm_b32 v2, v0, v0, s0
+; GCN-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll b/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
index 94e9395dd90e9..1bad81ee5e513 100644
--- a/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
@@ -11,13 +11,11 @@ define protected amdgpu_kernel void @add(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_add v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_add v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
@@ -33,13 +31,11 @@ define protected amdgpu_kernel void @sub(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_sub v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_sub v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw sub ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -55,13 +51,11 @@ define protected amdgpu_kernel void @and(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_and v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_and v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw and ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -77,13 +71,11 @@ define protected amdgpu_kernel void @or(ptr addrspace(1) %p, ptr addrspace(1) %q
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_or v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_or v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw or ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -99,13 +91,11 @@ define protected amdgpu_kernel void @xor(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_xor v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_xor v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw xor ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -137,9 +127,7 @@ define protected amdgpu_kernel void @nand(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_cbranch_execnz .LBB5_1
; CHECK-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, v[2:3]
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
@@ -156,13 +144,11 @@ define protected amdgpu_kernel void @max_workgroup(ptr addrspace(1) %p, ptr addr
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_smax v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_smax v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -178,13 +164,11 @@ define protected amdgpu_kernel void @max(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_smax v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_smax v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -200,13 +184,11 @@ define protected amdgpu_kernel void @min_workgroup(ptr addrspace(1) %p, ptr addr
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_smin v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_smin v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -222,13 +204,11 @@ define protected amdgpu_kernel void @min(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_smin v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_smin v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -244,13 +224,11 @@ define protected amdgpu_kernel void @umax_workgroup(ptr addrspace(1) %p, ptr add
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_umax v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_umax v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -266,13 +244,11 @@ define protected amdgpu_kernel void @umax(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_umax v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_umax v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -288,13 +264,11 @@ define protected amdgpu_kernel void @umin_workgroup(ptr addrspace(1) %p, ptr add
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_umin v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_umin v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -310,13 +284,11 @@ define protected amdgpu_kernel void @umin(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_umin v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_umin v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -334,12 +306,10 @@ define protected amdgpu_kernel void @cmpxchg(ptr addrspace(1) %p, ptr addrspace(
; CHECK-NEXT: v_mov_b32_e32 v0, 2
; CHECK-NEXT: v_mov_b32_e32 v1, 1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_cmpswap v2, v2, v[0:1], s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
+; CHECK-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%agg = cmpxchg ptr addrspace(1) %p, i32 1, i32 2 monotonic monotonic
@@ -356,13 +326,11 @@ define protected amdgpu_kernel void @xchg(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_swap v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_swap v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw xchg ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
@@ -378,13 +346,11 @@ define protected amdgpu_kernel void @inc(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_inc v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_inc v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw uinc_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -400,13 +366,11 @@ define protected amdgpu_kernel void @dec(ptr addrspace(1) %p, ptr addrspace(1) %
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_atomic_dec v2, v0, v1, s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_atomic_dec v0, v0, v1, s[0:1] glc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%n32 = atomicrmw udec_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -438,11 +402,9 @@ define protected amdgpu_kernel void @fadd(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_cbranch_execnz .LBB18_1
; CHECK-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%f32 = atomicrmw fadd ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -475,11 +437,9 @@ define protected amdgpu_kernel void @fsub(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: s_cbranch_execnz .LBB19_1
; CHECK-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%f32 = atomicrmw fsub ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
@@ -499,12 +459,10 @@ define protected amdgpu_kernel void @fmin(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: v_mov_b32_e32 v1, 0x3ff00000
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_atomic_min_f64 v[0:1], v2, v[0:1], s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
@@ -525,12 +483,10 @@ define protected amdgpu_kernel void @fmax(ptr addrspace(1) %p, ptr addrspace(1)
; CHECK-NEXT: v_mov_b32_e32 v1, 0x3ff00000
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_atomic_max_f64 v[0:1], v2, v[0:1], s[0:1] glc
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
+; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[2:3]
; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm
%f64 = atomicrmw fmax ptr addrspace(1) %p, double 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
index 16a998bb5e9bf..61a41de5cebed 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
@@ -461,23 +461,37 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9
; ALIGNED-SDAG-NEXT: ds_read_u8 v11, v0 offset:10
; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:11
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(6)
+; ALIGNED-SDAG-NEXT: v_lshlrev_b32_e32 v6, 8, v6
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: v_lshlrev_b32_e32 v12, 8, v8
+; ALIGNED-SDAG-NEXT: v_or_b32_e32 v6, v6, v5
+; ALIGNED-SDAG-NEXT: v_or_b32_sdwa v12, v12, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; ALIGNED-SDAG-NEXT: v_or_b32_e32 v6, v12, v6
+; ALIGNED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; ALIGNED-SDAG-NEXT: v_lshlrev_b32_e32 v12, 8, v4
+; ALIGNED-SDAG-NEXT: v_or_b32_e32 v2, v2, v1
+; ALIGNED-SDAG-NEXT: v_or_b32_sdwa v12, v12, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; ALIGNED-SDAG-NEXT: v_or_b32_e32 v2, v12, v2
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v12, s1
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v9 offset:8
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v10 offset:9
-; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v5 offset:4
-; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v6 offset:5
-; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v1
-; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v2 offset:1
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v3 offset:2
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v4 offset:3
+; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v1
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v7 offset:6
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v8 offset:7
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
+; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v5 offset:4
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(9)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v11 offset:10
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(9)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v0 offset:11
+; ALIGNED-SDAG-NEXT: v_lshrrev_b32_e32 v0, 8, v2
+; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v0 offset:1
+; ALIGNED-SDAG-NEXT: v_lshrrev_b32_e32 v0, 8, v6
+; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v0 offset:5
; ALIGNED-SDAG-NEXT: s_endpgm
;
; ALIGNED-GISEL-LABEL: ds12align1:
@@ -598,20 +612,19 @@ define amdgpu_kernel void @ds12align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0
; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:8
-; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4
-; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:2
+; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:2
+; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:4
; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:10
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:8
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4
+; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:2
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:2
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:6
+; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:4
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v0 offset:10
; ALIGNED-SDAG-NEXT: s_endpgm
@@ -820,12 +833,12 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v13 offset:12
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v14 offset:13
-; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v1
-; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v2 offset:1
-; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v5 offset:4
-; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v6 offset:5
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v9 offset:8
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v10 offset:9
+; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v5 offset:4
+; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v6 offset:5
+; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v1
+; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v2 offset:1
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v3 offset:2
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v4 offset:3
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v7 offset:6
@@ -986,19 +999,20 @@ define amdgpu_kernel void @ds16align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:2
; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4
; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:12
-; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6
-; ALIGNED-SDAG-NEXT: ds_read_u16 v7, v0 offset:8
+; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:8
+; ALIGNED-SDAG-NEXT: ds_read_u16 v7, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u16 v8, v0 offset:10
; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:14
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:12
-; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:8
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v7 offset:8
+; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:2
-; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:6
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
+; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v7 offset:6
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v8 offset:10
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index 176dffac4abc5..81c02bbe36d12 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -114,16 +114,16 @@ define <4 x i16> @vec_8xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1)
; GFX9-NEXT: global_load_dwordx4 v[2:5], v[0:1], off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: .LBB0_5: ; %exit
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, 15, v3 op_sel_hi:[0,0]
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_or_b32_e32 v1, 0xffff8000, v0
-; GFX9-NEXT: v_or_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 15, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 15, v3 op_sel_hi:[0,0]
; GFX9-NEXT: v_or_b32_e32 v2, 0xffff8000, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v3, 0xffff8000, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
-; GFX9-NEXT: v_perm_b32 v1, v3, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: vec_8xi16_extract_4xi16:
@@ -191,8 +191,8 @@ define <4 x i16> @vec_8xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, 0xffff8000, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, 0xffff8000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, 0xffff8000, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0xffff8000, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0xffff8000, v3
@@ -799,16 +799,16 @@ define <4 x i16> @vec_16xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
; GFX9-NEXT: .LBB3_5: ; %exit
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, 15, v5 op_sel_hi:[0,0]
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_or_b32_e32 v1, 0xffff8000, v0
-; GFX9-NEXT: v_or_b32_sdwa v2, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 15, v4 op_sel_hi:[0,1]
-; GFX9-NEXT: v_or_b32_e32 v3, 0xffff8000, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 15, v5 op_sel_hi:[0,0]
+; GFX9-NEXT: v_or_b32_e32 v2, 0xffff8000, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v3, 0xffff8000, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4
+; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: vec_16xi16_extract_4xi16:
@@ -884,8 +884,8 @@ define <4 x i16> @vec_16xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, 0xffff8000, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, 0xffff8000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, 0xffff8000, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0xffff8000, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0xffff8000, v3
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_to_subreg.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_to_subreg.ll
new file mode 100644
index 0000000000000..802dc0bfeb4fa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_to_subreg.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s -stop-after=amdgpu-isel | FileCheck %s
+
+; An extract_subvector that names a whole number of 32-bit registers starting on
+; a register boundary is selected to a single subregister copy. Everything else
+; keeps the elementwise lowering, which isel turns into a REG_SEQUENCE of the
+; individual registers.
+
+define amdgpu_kernel void @extract_v2i32_from_v4i32_at2(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v2i32_from_v4i32_at2
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_128, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY %11.sub2_sub3
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR killed [[V_MOV_B32_e32_]], killed [[COPY1]], killed %12, 0, 0, implicit $exec :: (volatile store (s64) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <4 x i32> asm "; def $0", "=v"()
+ %sub = shufflevector <4 x i32> %in, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+ store volatile <2 x i32> %sub, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @extract_v4i32_from_v8i32_at4(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v4i32_from_v8i32_at4
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_256, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY %11.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed [[V_MOV_B32_e32_]], killed [[COPY1]], killed %12, 0, 0, implicit $exec :: (volatile store (s128) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <8 x i32> asm "; def $0", "=v"()
+ %sub = shufflevector <8 x i32> %in, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ store volatile <4 x i32> %sub, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @extract_v2i64_from_v8i64_at2(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v2i64_from_v8i64_at2
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_512, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY %11.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed [[V_MOV_B32_e32_]], killed [[COPY1]], killed %12, 0, 0, implicit $exec :: (volatile store (s128) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <8 x i64> asm "; def $0", "=v"()
+ %sub = shufflevector <8 x i64> %in, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+ store volatile <2 x i64> %sub, ptr addrspace(1) %out
+ ret void
+}
+
+; 16-bit elements work as long as the piece is register aligned.
+define amdgpu_kernel void @extract_v4i16_from_v8i16_at4(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v4i16_from_v8i16_at4
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_128, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY %11.sub2_sub3
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR killed [[V_MOV_B32_e32_]], killed [[COPY1]], killed %12, 0, 0, implicit $exec :: (volatile store (s64) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <8 x i16> asm "; def $0", "=v"()
+ %sub = shufflevector <8 x i16> %in, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ store volatile <4 x i16> %sub, ptr addrspace(1) %out
+ ret void
+}
+
+; Starting halfway into a register is not a subregister reference.
+define amdgpu_kernel void @extract_v4i16_from_v8i16_at3(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v4i16_from_v8i16_at3
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_128, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY %11.sub1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY %11.sub2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY %11.sub3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY killed [[COPY3]]
+ ; CHECK-NEXT: [[V_ALIGNBIT_B32_opsel_e64_:%[0-9]+]]:vgpr_32 = V_ALIGNBIT_B32_opsel_e64 0, killed [[COPY4]], 0, killed [[COPY2]], 0, 16, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY %11.sub2_sub3
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
+ ; CHECK-NEXT: [[V_ALIGNBIT_B32_opsel_e64_1:%[0-9]+]]:vgpr_32 = V_ALIGNBIT_B32_opsel_e64 0, killed [[COPY6]], 0, killed [[COPY1]], 0, 16, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_ALIGNBIT_B32_opsel_e64_1]], %subreg.sub0, killed [[V_ALIGNBIT_B32_opsel_e64_]], %subreg.sub1
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR killed [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE]], killed %12, 0, 0, implicit $exec :: (volatile store (s64) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <8 x i16> asm "; def $0", "=v"()
+ %sub = shufflevector <8 x i16> %in, <8 x i16> poison, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+ store volatile <4 x i16> %sub, ptr addrspace(1) %out
+ ret void
+}
+
+; 9 registers wide: the register file has no subregister index of that width, so
+; this has to be built out of the individual registers.
+define amdgpu_kernel void @extract_v9i32_from_v16i32_at0(ptr addrspace(1) %out) {
+ ; CHECK-LABEL: name: extract_v9i32_from_v16i32_at0
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %12:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: INLINEASM &"; def $0", attdialect, regdef:VReg_512, def %11
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY %11.sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY %11.sub2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY %11.sub1
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY %11.sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY %11.sub7
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY %11.sub6
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY %11.sub5
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY %11.sub4
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY %11.sub8
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_]], killed [[COPY9]], %12, 32, 0, implicit $exec :: (volatile store (s32) into %ir.out.load + 32, align 32, addrspace 1)
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE]], %12, 16, 0, implicit $exec :: (volatile store (s128) into %ir.out.load + 16, addrspace 1)
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, killed [[COPY2]], %subreg.sub2, killed [[COPY1]], %subreg.sub3
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE1]], %12, 0, 0, implicit $exec :: (volatile store (s128) into %ir.out.load, align 64, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %in = call <16 x i32> asm "; def $0", "=v"()
+ %sub = shufflevector <16 x i32> %in, <16 x i32> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+ store volatile <9 x i32> %sub, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
index 68a867fe89174..2eb05e21be08d 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
@@ -14,15 +14,12 @@ define amdgpu_hs void @main(ptr addrspace(6) inreg %arg) {
; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
; GCN-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) poison`, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
- ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_96 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1, killed [[COPY1]], %subreg.sub2
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE]]
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0_sub1_sub2
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_96 = COPY [[COPY1]]
; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
; GCN-NEXT: [[DEF3:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
- ; GCN-NEXT: BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96) into `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY2]], [[COPY3]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96) into `ptr addrspace(8) poison`, align 1, addrspace 8)
; GCN-NEXT: S_ENDPGM 0
main_body:
%tmp25 = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) poison, i32 poison, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll b/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
index 061de86de64ba..7afd8b084add6 100644
--- a/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
+++ b/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
@@ -10,9 +10,9 @@ define amdgpu_kernel void @fence_barrier_latency_test(ptr addrspace(1) %global_p
; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x2
-; CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x40 nv
; CHECK-NEXT: s_load_b96 s[16:18], s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b256 s[8:15], s[4:5], 0x20 nv
+; CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x40 nv
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v28, s18
; CHECK-NEXT: tensor_load_to_lds s[0:3], s[8:15]
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f64.ll b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
index 79f7ba7ee88d6..17895ef14037f 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
@@ -1,8 +1,8 @@
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=FUNC,GCN,FMA_F64 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMA_F64 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu9.0a -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMAC_F64 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu9.0a -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMAC_F64,GFX90A %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMA_F64 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.50 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMAC_F64 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.50 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=FUNC,GCN,FMAC_F64,GFX1250 %s
declare double @llvm.fma.f64(double, double, double) nounwind readnone
declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
@@ -25,8 +25,13 @@ define amdgpu_kernel void @fma_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
; FUNC-LABEL: {{^}}fma_v2f64:
; FMA_F64: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
; FMA_F64: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; On gfx90a the addend is a subregister of the loaded tuple and its other
+; half is still live, so two-address cannot tie every accumulator and falls
+; back to the VOP3 form for one of each pair.
+; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @fma_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
%r0 = load <2 x double>, ptr addrspace(1) %in1
@@ -42,10 +47,17 @@ define amdgpu_kernel void @fma_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %in
; FMA_F64: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
; FMA_F64: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
; FMA_F64: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
-; FMAC_F64: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; On gfx90a the addend is a subregister of the loaded tuple and its other
+; half is still live, so two-address cannot tie every accumulator and falls
+; back to the VOP3 form for one of each pair.
+; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; GFX1250: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @fma_v4f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
%r0 = load <4 x double>, ptr addrspace(1) %in1
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 56415ef44c527..d3f805ba10242 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -494,34 +494,34 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-MUBUF-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e64 v6, s[4:5], 28, v0
-; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 28, v0
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[6:9], v[4:5] offset:112
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[10:13], v[0:1] offset:12
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[14:17], v[4:5] offset:80
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[18:21], v[4:5] offset:96
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[22:25], v[4:5] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[26:29], v[4:5] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: ; kill: killed $vgpr0 killed $vgpr1
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[34:37], v[4:5] offset:32
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
-; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
+; GFX90A-SDAG-MUBUF-NEXT: s_movk_i32 s4, 0x88
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 48, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dword v[4:5], v50
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dword v[0:1], v8
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[10:13]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx2 v[2:3], v[6:7] offset:128
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[4:5], v[14:17] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[4:5], v[18:21] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[4:5], v[26:29] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[30:33] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[4:5], v[34:37]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[48:51] offset:16
; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-SDAG-MUBUF-NEXT: s_setpc_b64 s[30:31]
;
@@ -532,34 +532,34 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-FLATSCR-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e64 v6, s[0:1], 28, v0
-; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 28, v0
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[6:9], v[4:5] offset:112
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[10:13], v[0:1] offset:12
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[14:17], v[4:5] offset:80
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[18:21], v[4:5] offset:96
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[22:25], v[4:5] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[26:29], v[4:5] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: ; kill: killed $vgpr0 killed $vgpr1
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[34:37], v[4:5] offset:32
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
-; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
+; GFX90A-SDAG-FLATSCR-NEXT: s_movk_i32 s0, 0x88
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 48, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dword v[4:5], v50
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dword v[0:1], v8
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[10:13]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx2 v[2:3], v[6:7] offset:128
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[4:5], v[14:17] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[4:5], v[18:21] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[4:5], v[26:29] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[30:33] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[4:5], v[34:37]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[48:51] offset:16
; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-SDAG-FLATSCR-NEXT: s_setpc_b64 s[30:31]
;
@@ -570,41 +570,41 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX10-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX10-SDAG-NEXT: s_clause 0x8
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[36:37] offset:80
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:96
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[36:37] offset:64
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[36:37] offset:16
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[36:37] offset:32
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:12
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[36:37]
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:112
-; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX10-SDAG-NEXT: v_add_co_u32 v35, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v36, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[35:36] offset:112
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_clause 0x7
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[7:10], v[0:1] offset:12
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[11:14], v[35:36] offset:80
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[15:18], v[35:36] offset:96
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[19:22], v[35:36] offset:48
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[23:26], v[35:36] offset:64
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[27:30], v[35:36] offset:16
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[31:34], v[35:36] offset:32
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[35:38], v[35:36]
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x88, v2
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX10-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX10-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, v2, 48
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
+; GFX10-SDAG-NEXT: flat_store_dword v[0:1], v6
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:64
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[7:10]
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[11:14] offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[15:18] offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:32
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[19:22] offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[23:26] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[28:31]
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[27:30] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:16
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[31:34]
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dword v[48:49], v38
-; GFX10-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[36:37] offset:128
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[35:38] offset:16
+; GFX10-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[4:5] offset:128
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -613,52 +613,52 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x5c
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[2:3]
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x4c
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[2:3]
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x7c
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[2:3]
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x6c
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[0:1], 0, s[2:3]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 28
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, 60
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, 44
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x5c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x4c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x7c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x6c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x8c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 60
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, 44
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[18:21], v[16:17]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[22:25], v[12:13]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[14:15]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[22:25], v[0:1] offset:12
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[12:13]
; GFX942-SDAG-NEXT: ; kill: killed $vgpr12_vgpr13
-; GFX942-SDAG-NEXT: ; kill: killed $vgpr14_vgpr15
; GFX942-SDAG-NEXT: ; kill: killed $vgpr16_vgpr17
; GFX942-SDAG-NEXT: s_nop 0
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[8:9]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[30:33], v[10:11]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[48:51], v[6:7]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[52:55], v[0:1] offset:12
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x8c
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[14:15]
+; GFX942-SDAG-NEXT: s_nop 0
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[30:33], v[8:9]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[34:37], v[10:11]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[52:55], v[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 28
; GFX942-SDAG-NEXT: flat_load_dwordx4 a[0:3], v[0:1]
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x60
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x50
-; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x88
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x88
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, s[0:1]
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, s[2:3]
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[2:3], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[2:3], 0, 48
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[2:3], 0, s[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, 48
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[48:51]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[52:55]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:16
-; GFX942-SDAG-NEXT: flat_store_dword v[10:11], a2
-; GFX942-SDAG-NEXT: flat_store_dwordx2 v[2:3], a[0:1] offset:128
+; GFX942-SDAG-NEXT: flat_store_dword v[0:1], v20
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[22:25]
+; GFX942-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[18:19] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:64
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[30:33]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[52:55] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[48:51]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], a[0:3] offset:16
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -671,38 +671,44 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT: v_add_co_u32 v35, vcc_lo, v0, 28
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
-; GFX11-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v36, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[35:36] offset:112
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
-; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX11-SDAG-NEXT: s_clause 0x7
+; GFX11-SDAG-NEXT: flat_load_b128 v[7:10], v[0:1] offset:12
+; GFX11-SDAG-NEXT: flat_load_b128 v[11:14], v[35:36] offset:80
+; GFX11-SDAG-NEXT: flat_load_b128 v[15:18], v[35:36] offset:96
+; GFX11-SDAG-NEXT: flat_load_b128 v[19:22], v[35:36] offset:64
+; GFX11-SDAG-NEXT: flat_load_b128 v[23:26], v[35:36] offset:32
+; GFX11-SDAG-NEXT: flat_load_b128 v[27:30], v[35:36] offset:16
+; GFX11-SDAG-NEXT: flat_load_b128 v[31:34], v[35:36]
+; GFX11-SDAG-NEXT: flat_load_b128 v[35:38], v[35:36] offset:48
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x88, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX11-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX11-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, v2, 48
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
-; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
-; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
-; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX11-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX11-SDAG-NEXT: flat_store_b32 v[0:1], v6
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[7:10]
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[48:49], v[11:14] offset:48
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[48:49], v[15:18] offset:64
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[48:49], v[19:22] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[48:49], v[23:26]
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[27:30] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[31:34] offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX11-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX11-SDAG-NEXT: flat_store_b64 v[2:3], v[4:5] offset:128
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -720,40 +726,46 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX12-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX12-SDAG-NEXT: v_add_co_u32 v35, vcc_lo, v0, 28
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX12-SDAG-NEXT: s_clause 0x7
-; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX12-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX12-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX12-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX12-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX12-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
-; GFX12-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX12-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v36, null, 0, v1, vcc_lo
+; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[35:36] offset:112
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX12-SDAG-NEXT: s_clause 0x7
+; GFX12-SDAG-NEXT: flat_load_b128 v[7:10], v[0:1] offset:12
+; GFX12-SDAG-NEXT: flat_load_b128 v[11:14], v[35:36] offset:80
+; GFX12-SDAG-NEXT: flat_load_b128 v[15:18], v[35:36] offset:96
+; GFX12-SDAG-NEXT: flat_load_b128 v[19:22], v[35:36] offset:64
+; GFX12-SDAG-NEXT: flat_load_b128 v[23:26], v[35:36] offset:32
+; GFX12-SDAG-NEXT: flat_load_b128 v[27:30], v[35:36] offset:16
+; GFX12-SDAG-NEXT: flat_load_b128 v[31:34], v[35:36]
+; GFX12-SDAG-NEXT: flat_load_b128 v[35:38], v[35:36] offset:48
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x88, v2
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX12-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX12-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, v2, 48
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX12-SDAG-NEXT: s_clause 0x7
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
-; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
-; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
-; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX12-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX12-SDAG-NEXT: flat_store_b32 v[0:1], v6
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x708
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[7:10]
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x608
+; GFX12-SDAG-NEXT: flat_store_b128 v[48:49], v[11:14] offset:48
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x508
+; GFX12-SDAG-NEXT: flat_store_b128 v[48:49], v[15:18] offset:64
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x408
+; GFX12-SDAG-NEXT: flat_store_b128 v[48:49], v[19:22] offset:32
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x308
+; GFX12-SDAG-NEXT: flat_store_b128 v[48:49], v[23:26]
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x208
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[27:30] offset:32
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x108
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[31:34] offset:16
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x8
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[4:5] offset:128
; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 30589cc8b7f77..2391f6cc5189d 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -2737,23 +2737,23 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: s_mov_b32 s4, s10
; GFX950-SDAG-NEXT: s_mov_b32 s5, s11
; GFX950-SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
-; GFX950-SDAG-NEXT: s_mov_b32 s1, s9
; GFX950-SDAG-NEXT: s_mov_b32 s0, s8
+; GFX950-SDAG-NEXT: s_mov_b32 s1, s9
; GFX950-SDAG-NEXT: s_movk_i32 s4, 0x7e00
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s5, v3
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s6, v2
-; GFX950-SDAG-NEXT: s_and_b32 s9, s5, 0x1ff
-; GFX950-SDAG-NEXT: s_or_b32 s6, s9, s6
-; GFX950-SDAG-NEXT: s_cselect_b32 s6, 1, 0
-; GFX950-SDAG-NEXT: s_lshr_b32 s9, s5, 8
-; GFX950-SDAG-NEXT: s_bfe_u32 s10, s5, 0xb0014
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s8, v3
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v2
+; GFX950-SDAG-NEXT: s_and_b32 s9, s8, 0x1ff
+; GFX950-SDAG-NEXT: s_or_b32 s7, s9, s7
+; GFX950-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s9, s8, 8
+; GFX950-SDAG-NEXT: s_bfe_u32 s10, s8, 0xb0014
; GFX950-SDAG-NEXT: s_and_b32 s9, s9, 0xffe
; GFX950-SDAG-NEXT: s_sub_i32 s11, 0x3f1, s10
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX950-SDAG-NEXT: s_or_b32 s6, s9, s6
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX950-SDAG-NEXT: s_or_b32 s7, s9, s7
; GFX950-SDAG-NEXT: v_med3_i32 v0, s11, 0, 13
-; GFX950-SDAG-NEXT: s_or_b32 s9, s6, 0x1000
+; GFX950-SDAG-NEXT: s_or_b32 s9, s7, 0x1000
; GFX950-SDAG-NEXT: v_readfirstlane_b32 s11, v0
; GFX950-SDAG-NEXT: s_lshr_b32 s12, s9, s11
; GFX950-SDAG-NEXT: s_lshl_b32 s11, s12, s11
@@ -2762,7 +2762,7 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: s_addk_i32 s10, 0xfc10
; GFX950-SDAG-NEXT: s_lshl_b32 s11, s10, 12
; GFX950-SDAG-NEXT: s_or_b32 s9, s12, s9
-; GFX950-SDAG-NEXT: s_or_b32 s11, s6, s11
+; GFX950-SDAG-NEXT: s_or_b32 s11, s7, s11
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s10, 1
; GFX950-SDAG-NEXT: s_cselect_b32 s9, s9, s11
; GFX950-SDAG-NEXT: s_and_b32 s11, s9, 7
@@ -2775,24 +2775,24 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: s_add_i32 s9, s9, s11
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s10, 31
; GFX950-SDAG-NEXT: s_cselect_b32 s9, s9, 0x7c00
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s6, 0
-; GFX950-SDAG-NEXT: s_cselect_b32 s6, s4, 0x7c00
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s7, 0
+; GFX950-SDAG-NEXT: s_cselect_b32 s7, s4, 0x7c00
; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; GFX950-SDAG-NEXT: s_cselect_b32 s6, s6, s9
-; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 16
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v1
-; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
-; GFX950-SDAG-NEXT: s_and_b32 s9, s7, 0x1ff
-; GFX950-SDAG-NEXT: s_or_b32 s5, s5, s6
-; GFX950-SDAG-NEXT: s_or_b32 s6, s9, s8
-; GFX950-SDAG-NEXT: s_cselect_b32 s6, 1, 0
-; GFX950-SDAG-NEXT: s_lshr_b32 s8, s7, 8
-; GFX950-SDAG-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; GFX950-SDAG-NEXT: s_cselect_b32 s7, s7, s9
+; GFX950-SDAG-NEXT: s_lshr_b32 s8, s8, 16
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s6, v1
+; GFX950-SDAG-NEXT: s_and_b32 s8, s8, 0x8000
+; GFX950-SDAG-NEXT: s_and_b32 s9, s6, 0x1ff
+; GFX950-SDAG-NEXT: s_or_b32 s7, s8, s7
+; GFX950-SDAG-NEXT: s_or_b32 s5, s9, s5
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s8, s6, 8
+; GFX950-SDAG-NEXT: s_bfe_u32 s9, s6, 0xb0014
; GFX950-SDAG-NEXT: s_and_b32 s8, s8, 0xffe
; GFX950-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
-; GFX950-SDAG-NEXT: s_or_b32 s6, s8, s6
+; GFX950-SDAG-NEXT: s_or_b32 s5, s8, s5
; GFX950-SDAG-NEXT: v_med3_i32 v0, s10, 0, 13
-; GFX950-SDAG-NEXT: s_or_b32 s8, s6, 0x1000
+; GFX950-SDAG-NEXT: s_or_b32 s8, s5, 0x1000
; GFX950-SDAG-NEXT: v_readfirstlane_b32 s10, v0
; GFX950-SDAG-NEXT: s_lshr_b32 s11, s8, s10
; GFX950-SDAG-NEXT: s_lshl_b32 s10, s11, s10
@@ -2801,7 +2801,7 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: s_addk_i32 s9, 0xfc10
; GFX950-SDAG-NEXT: s_lshl_b32 s10, s9, 12
; GFX950-SDAG-NEXT: s_or_b32 s8, s11, s8
-; GFX950-SDAG-NEXT: s_or_b32 s10, s6, s10
+; GFX950-SDAG-NEXT: s_or_b32 s10, s5, s10
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 1
; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, s10
; GFX950-SDAG-NEXT: s_and_b32 s10, s8, 7
@@ -2814,14 +2814,14 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: s_add_i32 s8, s8, s10
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 31
; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s6, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s5, 0
; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s8
-; GFX950-SDAG-NEXT: s_lshr_b32 s6, s7, 16
-; GFX950-SDAG-NEXT: s_and_b32 s6, s6, 0x8000
-; GFX950-SDAG-NEXT: s_or_b32 s4, s6, s4
-; GFX950-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s5
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s6, 16
+; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
+; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX950-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s7
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, s4
; GFX950-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 433dc96366e62..1795338ed89b8 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -371,16 +371,16 @@ define void @freeze_v5i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v5i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v8, v[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v8, v[8:9]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v8
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v8
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -657,15 +657,15 @@ define void @freeze_v7i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v10, v[0:1], s[4:7], 0 addr64 offset:24
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dword v8, v[0:1], s[4:7], 0 addr64 offset:24
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX6-SDAG-NEXT: buffer_store_dword v10, v[2:3], s[4:7], 0 addr64 offset:24
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[8:9], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dword v8, v[2:3], s[4:7], 0 addr64 offset:24
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -694,12 +694,12 @@ define void @freeze_v7i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx3 v[8:10], v[0:1], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX7-SDAG-NEXT: buffer_store_dwordx3 v[8:10], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx3 v[8:10], v[0:1], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX7-SDAG-NEXT: buffer_store_dwordx3 v[8:10], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -750,77 +750,41 @@ define void @freeze_v7i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GFX900-SDAG-LABEL: freeze_v7i32:
-; GFX8-GFX900-SDAG: ; %bb.0:
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: freeze_v7i32:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-GISEL-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: freeze_v7i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: freeze_v7i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-GISEL-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: freeze_v7i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX9-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-LABEL: freeze_v7i32:
-; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: global_load_b96 v[8:10], v[0:1], off offset:16
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[8:10], off offset:16
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: freeze_v7i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-NEXT: global_load_dwordx3 v[8:10], v[0:1], off offset:16
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_store_dwordx3 v[2:3], v[8:10], off offset:16
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: freeze_v7i32:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-GISEL-NEXT: global_load_b96 v[8:10], v[0:1], off offset:16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: global_store_b96 v[2:3], v[8:10], off offset:16
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: freeze_v7i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-NEXT: global_load_b96 v[8:10], v[0:1], off offset:16
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b96 v[2:3], v[8:10], off offset:16
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%a = load <7 x i32>, ptr addrspace(1) %ptra, align 4
%freeze = freeze <7 x i32> %a
store <7 x i32> %freeze, ptr addrspace(1) %ptrb, align 4
@@ -1007,15 +971,15 @@ define void @freeze_v9i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v12, v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX6-SDAG-NEXT: buffer_store_dword v12, v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1044,15 +1008,15 @@ define void @freeze_v9i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dword v12, v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX7-SDAG-NEXT: buffer_store_dword v12, v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1077,23 +1041,23 @@ define void @freeze_v9i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v9i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v14, v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v14, v[12:13]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v14
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[12:13], v14
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1120,95 +1084,50 @@ define void @freeze_v9i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GFX900-SDAG-LABEL: freeze_v9i32:
-; GFX8-GFX900-SDAG: ; %bb.0:
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dword v12, v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v12, off offset:32
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: freeze_v9i32:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX9-GISEL-NEXT: global_load_dword v12, v[0:1], off offset:32
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-GISEL-NEXT: global_store_dword v[2:3], v12, off offset:32
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: freeze_v9i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_clause 0x2
-; GFX10-SDAG-NEXT: global_load_dword v12, v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dword v[2:3], v12, off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: freeze_v9i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_clause 0x2
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX10-GISEL-NEXT: global_load_dword v12, v[0:1], off offset:32
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: global_store_dword v[2:3], v12, off offset:32
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: freeze_v9i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX9-NEXT: global_load_dword v12, v[0:1], off offset:32
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: global_store_dword v[2:3], v12, off offset:32
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-LABEL: freeze_v9i32:
-; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_clause 0x2
-; GFX11-SDAG-NEXT: global_load_b32 v12, v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v12, off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
-; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: freeze_v9i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x2
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX10-NEXT: global_load_dword v12, v[0:1], off offset:32
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_store_dword v[2:3], v12, off offset:32
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: freeze_v9i32:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_clause 0x2
-; GFX11-GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-GISEL-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
-; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:32
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: global_store_b32 v[2:3], v0, off offset:32
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: freeze_v9i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:32
+; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off offset:32
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%a = load <9 x i32>, ptr addrspace(1) %ptra, align 4
%freeze = freeze <9 x i32> %a
store <9 x i32> %freeze, ptr addrspace(1) %ptrb, align 4
@@ -1394,18 +1313,18 @@ define void @freeze_v11i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v14, v[0:1], s[4:7], 0 addr64 offset:40
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[12:13], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dword v12, v[0:1], s[4:7], 0 addr64 offset:40
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dword v14, v[2:3], s[4:7], 0 addr64 offset:40
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[12:13], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dword v12, v[2:3], s[4:7], 0 addr64 offset:40
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1742,17 +1661,17 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
; GFX6-SDAG-NEXT: buffer_load_dword v16, v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dword v16, v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1785,17 +1704,17 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
; GFX7-SDAG-NEXT: buffer_load_dword v16, v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dword v16, v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1823,25 +1742,24 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v13i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v18, v[4:5]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v18, v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v18
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
@@ -1884,17 +1802,17 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GFX900-SDAG-NEXT: global_load_dword v16, v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v16, off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1921,17 +1839,17 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x3
; GFX10-SDAG-NEXT: global_load_dword v16, v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dword v[2:3], v16, off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v13i32:
@@ -1956,18 +1874,18 @@ define void @freeze_v13i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x3
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
; GFX11-SDAG-NEXT: global_load_b32 v16, v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v16, off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off
+; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v16, off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:16
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v13i32:
@@ -2001,18 +1919,18 @@ define void @freeze_v14i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[16:17], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[16:17], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2044,18 +1962,18 @@ define void @freeze_v14i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[16:17], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[16:17], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2083,30 +2001,29 @@ define void @freeze_v14i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v14i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v14, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[14:15]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[16:17]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v19, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[18:19], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[18:19], v[12:15]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2140,113 +2057,59 @@ define void @freeze_v14i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GFX900-SDAG-LABEL: freeze_v14i32:
-; GFX8-GFX900-SDAG: ; %bb.0:
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: freeze_v14i32:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: freeze_v14i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_clause 0x3
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: freeze_v14i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_clause 0x3
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: freeze_v14i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX9-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX9-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-LABEL: freeze_v14i32:
-; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_clause 0x3
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b64 v[16:17], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:16
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[16:17], off offset:48
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:16
-; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: freeze_v14i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x3
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX10-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX10-NEXT: global_load_dwordx2 v[16:17], v[0:1], off offset:48
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[16:17], off offset:48
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: freeze_v14i32:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_clause 0x3
-; GFX11-GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-GISEL-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
-; GFX11-GISEL-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
-; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off offset:48
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off offset:48
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: freeze_v14i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x3
+; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
+; GFX11-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:48
+; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off offset:48
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%a = load <14 x i32>, ptr addrspace(1) %ptra, align 4
%freeze = freeze <14 x i32> %a
store <14 x i32> %freeze, ptr addrspace(1) %ptrb, align 4
@@ -2261,21 +2124,21 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v18, v[0:1], s[4:7], 0 addr64 offset:56
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[16:17], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dword v16, v[0:1], s[4:7], 0 addr64 offset:56
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dword v18, v[2:3], s[4:7], 0 addr64 offset:56
+; GFX6-SDAG-NEXT: buffer_store_dword v16, v[2:3], s[4:7], 0 addr64 offset:56
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[16:17], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2310,21 +2173,21 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dword v18, v[0:1], s[4:7], 0 addr64 offset:56
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[16:17], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dword v16, v[0:1], s[4:7], 0 addr64 offset:56
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dword v18, v[2:3], s[4:7], 0 addr64 offset:56
+; GFX7-SDAG-NEXT: buffer_store_dword v16, v[2:3], s[4:7], 0 addr64 offset:56
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[16:17], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2352,37 +2215,37 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v15i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v14, vcc, 56, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 56, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v18, v[14:15]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v18, v[12:13]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[16:17]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 56, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_nop 0
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[16:17], v18
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[6:7], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[12:15]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2417,113 +2280,59 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GFX900-SDAG-LABEL: freeze_v15i32:
-; GFX8-GFX900-SDAG: ; %bb.0:
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: freeze_v15i32:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
-; GFX9-GISEL-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX9-GISEL-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: freeze_v15i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_clause 0x3
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:16
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: freeze_v15i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_clause 0x3
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
-; GFX10-GISEL-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: freeze_v15i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX9-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX9-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-LABEL: freeze_v15i32:
-; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_clause 0x3
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b96 v[16:18], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:16
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[16:18], off offset:48
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:16
-; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: freeze_v15i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x3
+; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
+; GFX10-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX10-NEXT: global_load_dwordx3 v[16:18], v[0:1], off offset:48
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_store_dwordx3 v[2:3], v[16:18], off offset:48
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: freeze_v15i32:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_clause 0x3
-; GFX11-GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-GISEL-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
-; GFX11-GISEL-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
-; GFX11-GISEL-NEXT: global_load_b96 v[16:18], v[0:1], off offset:48
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(3)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(2)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1)
-; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: global_store_b96 v[2:3], v[16:18], off offset:48
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: freeze_v15i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x3
+; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
+; GFX11-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
+; GFX11-NEXT: global_load_b96 v[16:18], v[0:1], off offset:48
+; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b96 v[2:3], v[16:18], off offset:48
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%a = load <15 x i32>, ptr addrspace(1) %ptra, align 4
%freeze = freeze <15 x i32> %a
store <15 x i32> %freeze, ptr addrspace(1) %ptrb, align 4
@@ -2800,13 +2609,11 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v20, v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dword v20, v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -2815,6 +2622,8 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2849,13 +2658,11 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dword v20, v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dword v20, v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -2864,6 +2671,8 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2894,37 +2703,35 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v17i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 64, v0
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v20, v[10:11]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v19, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v20, v[16:17]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[18:19]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v20
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[2:3], v20
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2966,13 +2773,11 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v17i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dword v20, v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v20, off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_load_dword v20, v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -2981,6 +2786,8 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v20, off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3009,21 +2816,21 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x4
-; GFX10-SDAG-NEXT: global_load_dword v20, v[0:1], off offset:64
; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dword v20, v[0:1], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dword v[2:3], v20, off offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dword v[2:3], v20, off offset:64
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v17i32:
@@ -3051,21 +2858,21 @@ define void @freeze_v17i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x4
-; GFX11-SDAG-NEXT: global_load_b32 v20, v[0:1], off offset:64
; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v20, off offset:64
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v0, off offset:64
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v17i32:
@@ -3102,13 +2909,11 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -3117,6 +2922,8 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3151,13 +2958,11 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -3166,6 +2971,8 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3196,37 +3003,35 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v18i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 64, v0
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[20:21], v[10:11]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v19, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[20:21], v[16:17]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[18:19]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[20:21]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[20:21]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3268,13 +3073,11 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v18i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[20:21], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[20:21], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[20:21], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -3283,6 +3086,8 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[20:21], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3311,21 +3116,21 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x4
-; GFX10-SDAG-NEXT: global_load_dwordx2 v[20:21], v[0:1], off offset:64
; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx2 v[20:21], v[0:1], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[20:21], off offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[20:21], off offset:64
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v18i32:
@@ -3353,21 +3158,21 @@ define void @freeze_v18i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x4
-; GFX11-SDAG-NEXT: global_load_b64 v[20:21], v[0:1], off offset:64
; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[20:21], off offset:64
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v18i32:
@@ -3404,24 +3209,22 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dword v22, v[2:3], s[4:7], 0 addr64 offset:72
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dword v22, v[2:3], s[4:7], 0 addr64 offset:72
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3457,24 +3260,22 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dword v22, v[2:3], s[4:7], 0 addr64 offset:72
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dword v22, v[2:3], s[4:7], 0 addr64 offset:72
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3505,45 +3306,45 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v19i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s4, 0x48
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, s4, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v22, v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 64, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v22, vcc, 0x48, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e64 v8, s[4:5], 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v23, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v1, s[4:5]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[20:21], v[10:11]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s4, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v22, v[22:23]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[20:21]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[20:21], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_nop 0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[0:1], v22
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 0x48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[20:21]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[16:19]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[4:5], v22
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[20:21], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3587,13 +3388,11 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v19i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx3 v[20:22], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
-; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[20:22], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx3 v[20:22], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
@@ -3602,6 +3401,8 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[20:22], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3630,21 +3431,21 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x4
-; GFX10-SDAG-NEXT: global_load_dwordx3 v[20:22], v[0:1], off offset:64
; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx3 v[20:22], v[0:1], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[20:22], off offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[20:22], off offset:64
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v19i32:
@@ -3672,21 +3473,21 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x4
-; GFX11-SDAG-NEXT: global_load_b96 v[20:22], v[0:1], off offset:64
; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b96 v[20:22], v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[20:22], off offset:64
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[20:22], off offset:64
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v19i32:
@@ -3723,21 +3524,21 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3772,21 +3573,21 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3817,29 +3618,29 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v20i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 48, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 64, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_nop 0
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
@@ -3893,21 +3694,21 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v20i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -3936,21 +3737,21 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x4
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v20i32:
@@ -3978,21 +3779,21 @@ define void @freeze_v20i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x4
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:64
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:64
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:16
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:64
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v20i32:
@@ -4029,24 +3830,24 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: buffer_load_dword v24, v[0:1], s[4:7], 0 addr64 offset:80
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX6-SDAG-NEXT: buffer_store_dword v24, v[2:3], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4084,24 +3885,24 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: buffer_load_dword v24, v[0:1], s[4:7], 0 addr64 offset:80
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX7-SDAG-NEXT: buffer_store_dword v24, v[2:3], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4135,36 +3936,36 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v21i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s4, 0x50
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, s4, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dword v24, v[12:13]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s4, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
@@ -4223,24 +4024,24 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v21i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
; GFX8-GFX900-SDAG-NEXT: global_load_dword v24, v[0:1], off offset:80
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v24, off offset:80
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4273,23 +4074,23 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x5
; GFX10-SDAG-NEXT: global_load_dword v24, v[0:1], off offset:80
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5)
; GFX10-SDAG-NEXT: global_store_dword v[2:3], v24, off offset:80
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v21i32:
@@ -4320,24 +4121,24 @@ define void @freeze_v21i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x5
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
; GFX11-SDAG-NEXT: global_load_b32 v24, v[0:1], off offset:80
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:64
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v24, off offset:80
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:64
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off
+; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v24, off offset:80
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:16
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:64
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v21i32:
@@ -4377,24 +4178,24 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[24:25], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[24:25], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4432,24 +4233,24 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[24:25], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[24:25], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4483,43 +4284,44 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v22i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 64, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s4, 0x50
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, s4, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 32, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[24:25], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[10:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 0x50, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 48, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v22, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v23, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[20:21]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[22:23]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v24, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v25, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s4, v2
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[24:25], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_nop 0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[4:5], v[24:25]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 0x50, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[20:23]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[20:23]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[8:9], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4570,24 +4372,24 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v22i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[24:25], v[0:1], off offset:80
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[24:25], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[24:25], off offset:80
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4619,24 +4421,24 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x5
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:32
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:64
; GFX10-SDAG-NEXT: global_load_dwordx2 v[24:25], v[0:1], off offset:80
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[24:25], off offset:80
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:48
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[24:25], off offset:80
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v22i32:
@@ -4667,24 +4469,24 @@ define void @freeze_v22i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x5
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:64
-; GFX11-SDAG-NEXT: global_load_b64 v[24:25], v[0:1], off offset:80
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:32
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:48
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:64
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off offset:80
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:64
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[24:25], off offset:80
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:16
+; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off offset:80
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v22i32:
@@ -4724,30 +4526,30 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:96
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:80
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:112
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:96
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:96
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:112
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4791,30 +4593,30 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:96
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:80
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64 offset:112
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:96
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:96
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:112
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4854,59 +4656,59 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v30i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v28, vcc, 16, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v28, vcc, 0x70, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s4, 0x60
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s4, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 64, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 64, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s5, 0x70
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, s5, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[32:33], v[10:11]
+; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x50
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v30, vcc, 0x60, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e64 v8, s[4:5], s6, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v1, s[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v24, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v24, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v25, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x50
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, s6, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 48, v0
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s4, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[28:29]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[30:31]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v32, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[32:33], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_nop 0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s6, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s5, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 48, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[32:33]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 0x60, v2
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[20:23]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: s_mov_b64 s[4:5], vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 0x70, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v3, s[4:5]
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[20:23]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[24:27]
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[28:31]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[28:31]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[6:7], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -4976,30 +4778,31 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v30i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:96
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:64
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:80
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[32:33], v[0:1], off offset:112
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:96
+; GFX8-GFX900-SDAG-NEXT: s_nop 0
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:112
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:96
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:80
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[32:33], off offset:112
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:96
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[0:1], off offset:112
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5038,30 +4841,30 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x7
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:96
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:80
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:48
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:96
; GFX10-SDAG-NEXT: global_load_dwordx2 v[32:33], v[0:1], off offset:112
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:64
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:80
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:96
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6)
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[32:33], off offset:112
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:80
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:64
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:80
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:48
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:96
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[32:33], off offset:112
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v30i32:
@@ -5098,30 +4901,30 @@ define void @freeze_v30i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:96
-; GFX11-SDAG-NEXT: global_load_b64 v[32:33], v[0:1], off offset:112
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:64
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:80
-; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[24:27], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[28:31], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:64
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:80
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
+; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:48
+; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[24:27], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[28:31], v[0:1], off offset:96
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off offset:112
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:96
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(6)
-; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[32:33], off offset:112
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:80
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:64
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:80
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[24:27], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[24:27], off
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[28:31], off offset:96
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[28:31], off offset:16
+; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off offset:112
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v30i32:
@@ -5167,33 +4970,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:80
; GFX6-SDAG-NEXT: buffer_load_dword v34, v[0:1], s[4:7], 0 addr64 offset:120
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64 offset:16
; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:96
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:64
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:80
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX6-SDAG-NEXT: buffer_store_dword v34, v[2:3], s[4:7], 0 addr64 offset:120
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:32
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:96
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5238,33 +5038,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:80
; GFX7-SDAG-NEXT: buffer_load_dword v34, v[0:1], s[4:7], 0 addr64 offset:120
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64 offset:16
; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64
-; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:96
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:64
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:80
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64 offset:80
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
; GFX7-SDAG-NEXT: buffer_store_dword v34, v[2:3], s[4:7], 0 addr64 offset:120
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:32
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64 offset:16
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:96
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5304,66 +5101,69 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-LABEL: freeze_v31i32:
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s4, 0x60
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s4, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e64 v4, s[6:7], 64, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v32, vcc, 0x70, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e64 v5, s[6:7], 0, v1, s[6:7]
+; GFX8-GFX803-SDAG-NEXT: s_mov_b64 s[4:5], vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v28, vcc, 0x60, v0
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v24, vcc, 48, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v24, vcc, 16, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v25, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 64, v0
+; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x50
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, s6, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x70
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, s6, v0
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s5, 0x50
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[32:33], v[10:11]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v12, vcc, s5, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s7, 0x78
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v14, vcc, s7, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 48, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dword v34, v[14:15]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 32, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v17, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s7, 0x78
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v26, vcc, s7, v0
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v27, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v34, v[26:27]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v0
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[0:1]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s4, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e64 v33, vcc, 0, v1, s[4:5]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[32:33]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v32, vcc, 64, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s5, v2
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[32:33], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_nop 0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s6, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, s7, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s6, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 32, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v32, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, s7, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 48, v2
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 0x60, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
-; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[6:7], v34
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[32:33]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[20:23]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[16:19]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[24:27]
-; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
-; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[32:33], v[20:23]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[4:5], v34
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[24:27]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[28:31]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 0x70, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(8)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5432,30 +5232,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-LABEL: freeze_v31i32:
; GFX8-GFX900-SDAG: ; %bb.0:
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:96
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:64
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:96
; GFX8-GFX900-SDAG-NEXT: global_load_dwordx3 v[32:34], v[0:1], off offset:112
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:32
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:48
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off
-; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:96
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:64
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:80
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:80
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[32:34], off offset:112
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:48
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:32
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:48
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off offset:16
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:96
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:16
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx3 v[2:3], v[32:34], off offset:112
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5493,30 +5293,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_clause 0x7
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:96
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:64
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:80
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:32
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:48
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off offset:16
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:96
; GFX10-SDAG-NEXT: global_load_dwordx3 v[32:34], v[0:1], off offset:112
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:64
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:80
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:32
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:48
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[24:27], v[0:1], off
-; GFX10-SDAG-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:96
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6)
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[32:34], off offset:112
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:80
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:64
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off offset:80
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:48
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off offset:32
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[20:23], off offset:48
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off offset:16
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[24:27], off
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:96
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[28:31], off offset:16
+; GFX10-SDAG-NEXT: global_store_dwordx3 v[2:3], v[32:34], off offset:112
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: freeze_v31i32:
@@ -5553,30 +5353,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:96
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:64
+; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:80
+; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:32
+; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:48
+; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off
+; GFX11-SDAG-NEXT: global_load_b128 v[24:27], v[0:1], off offset:16
+; GFX11-SDAG-NEXT: global_load_b128 v[28:31], v[0:1], off offset:96
; GFX11-SDAG-NEXT: global_load_b96 v[32:34], v[0:1], off offset:112
-; GFX11-SDAG-NEXT: global_load_b128 v[8:11], v[0:1], off offset:64
-; GFX11-SDAG-NEXT: global_load_b128 v[12:15], v[0:1], off offset:80
-; GFX11-SDAG-NEXT: global_load_b128 v[16:19], v[0:1], off offset:32
-; GFX11-SDAG-NEXT: global_load_b128 v[20:23], v[0:1], off offset:48
-; GFX11-SDAG-NEXT: global_load_b128 v[24:27], v[0:1], off
-; GFX11-SDAG-NEXT: global_load_b128 v[28:31], v[0:1], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(7)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:96
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off offset:64
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(6)
-; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[32:34], off offset:112
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:80
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[8:11], off offset:64
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:32
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[12:15], off offset:80
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:48
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[16:19], off offset:32
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[20:23], off offset:48
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[24:27], off offset:16
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[24:27], off
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[28:31], off offset:96
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[28:31], off offset:16
+; GFX11-SDAG-NEXT: global_store_b96 v[2:3], v[32:34], off offset:112
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: freeze_v31i32:
@@ -11929,35 +11729,20 @@ define void @freeze_v2p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
}
define void @freeze_v3p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
-; GFX6-SDAG-LABEL: freeze_v3p3:
-; GFX6-SDAG: ; %bb.0:
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 8, v0
-; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT: ds_read_b32 v4, v2
-; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-SDAG-NEXT: ds_write_b32 v0, v4
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-SDAG-NEXT: ds_write_b64 v1, v[2:3]
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX6-GISEL-LABEL: freeze_v3p3:
-; GFX6-GISEL: ; %bb.0:
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
-; GFX6-GISEL-NEXT: ds_read_b64 v[2:3], v0
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 8, v0
-; GFX6-GISEL-NEXT: ds_read_b32 v0, v0
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-GISEL-NEXT: ds_write_b64 v1, v[2:3]
-; GFX6-GISEL-NEXT: v_add_i32_e32 v1, vcc, 8, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-GISEL-NEXT: ds_write_b32 v1, v0
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: freeze_v3p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 8, v0
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: ds_write_b64 v1, v[2:3]
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 8, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: freeze_v3p3:
; GFX7: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index dd9548c42e148..7ed6ce804db70 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -1442,6 +1442,7 @@ define <33 x i32> @v33i32_func_void() #0 {
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x8
+; GFX11-NEXT: buffer_load_b32 v33, off, s[0:3], 0 offset:128
; GFX11-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:112
; GFX11-NEXT: buffer_load_b128 v[5:8], off, s[0:3], 0 offset:96
; GFX11-NEXT: buffer_load_b128 v[9:12], off, s[0:3], 0 offset:80
@@ -1450,25 +1451,24 @@ define <33 x i32> @v33i32_func_void() #0 {
; GFX11-NEXT: buffer_load_b128 v[21:24], off, s[0:3], 0 offset:32
; GFX11-NEXT: buffer_load_b128 v[25:28], off, s[0:3], 0 offset:16
; GFX11-NEXT: buffer_load_b128 v[29:32], off, s[0:3], 0
-; GFX11-NEXT: buffer_load_b32 v33, off, s[0:3], 0 offset:128
; GFX11-NEXT: s_waitcnt vmcnt(8)
-; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:112
+; GFX11-NEXT: scratch_store_b32 v0, v33, off offset:128
; GFX11-NEXT: s_waitcnt vmcnt(7)
-; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:96
+; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:112
; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:96
; GFX11-NEXT: s_waitcnt vmcnt(5)
-; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:64
+; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:80
; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: scratch_store_b128 v0, v[17:20], off offset:48
+; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:64
; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:32
+; GFX11-NEXT: scratch_store_b128 v0, v[17:20], off offset:48
; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: scratch_store_b128 v0, v[25:28], off offset:16
+; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:32
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off
+; GFX11-NEXT: scratch_store_b128 v0, v[25:28], off offset:16
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b32 v0, v33, off offset:128
+; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) poison
%val = load <33 x i32>, ptr addrspace(1) %ptr
@@ -1659,6 +1659,7 @@ define { <32 x i32>, i32 } @struct_v32i32_i32_func_void() #0 {
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x8
+; GFX11-NEXT: buffer_load_b32 v33, off, s[0:3], 0 offset:128
; GFX11-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:112
; GFX11-NEXT: buffer_load_b128 v[5:8], off, s[0:3], 0 offset:96
; GFX11-NEXT: buffer_load_b128 v[9:12], off, s[0:3], 0 offset:80
@@ -1667,25 +1668,24 @@ define { <32 x i32>, i32 } @struct_v32i32_i32_func_void() #0 {
; GFX11-NEXT: buffer_load_b128 v[21:24], off, s[0:3], 0 offset:32
; GFX11-NEXT: buffer_load_b128 v[25:28], off, s[0:3], 0 offset:16
; GFX11-NEXT: buffer_load_b128 v[29:32], off, s[0:3], 0
-; GFX11-NEXT: buffer_load_b32 v33, off, s[0:3], 0 offset:128
; GFX11-NEXT: s_waitcnt vmcnt(8)
-; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:112
+; GFX11-NEXT: scratch_store_b32 v0, v33, off offset:128
; GFX11-NEXT: s_waitcnt vmcnt(7)
-; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:96
+; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:112
; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:96
; GFX11-NEXT: s_waitcnt vmcnt(5)
-; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:64
+; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:80
; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: scratch_store_b128 v0, v[17:20], off offset:48
+; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:64
; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:32
+; GFX11-NEXT: scratch_store_b128 v0, v[17:20], off offset:48
; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: scratch_store_b128 v0, v[25:28], off offset:16
+; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:32
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off
+; GFX11-NEXT: scratch_store_b128 v0, v[25:28], off offset:16
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b32 v0, v33, off offset:128
+; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off
; GFX11-NEXT: s_setpc_b64 s[30:31]
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) poison
%val = load { <32 x i32>, i32 }, ptr addrspace(1) %ptr
@@ -1883,8 +1883,8 @@ define { i32, <32 x i32> } @struct_i32_v32i32_func_void() #0 {
; GFX11-NEXT: buffer_load_b128 v[17:20], off, s[0:3], 0 offset:176
; GFX11-NEXT: buffer_load_b128 v[21:24], off, s[0:3], 0 offset:160
; GFX11-NEXT: buffer_load_b128 v[25:28], off, s[0:3], 0 offset:144
-; GFX11-NEXT: buffer_load_b128 v[29:32], off, s[0:3], 0 offset:128
; GFX11-NEXT: buffer_load_b32 v33, off, s[0:3], 0
+; GFX11-NEXT: buffer_load_b128 v[29:32], off, s[0:3], 0 offset:128
; GFX11-NEXT: s_waitcnt vmcnt(8)
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:240
; GFX11-NEXT: s_waitcnt vmcnt(7)
@@ -1900,9 +1900,9 @@ define { i32, <32 x i32> } @struct_i32_v32i32_func_void() #0 {
; GFX11-NEXT: s_waitcnt vmcnt(2)
; GFX11-NEXT: scratch_store_b128 v0, v[25:28], off offset:144
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off offset:128
-; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: scratch_store_b32 v0, v33, off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: scratch_store_b128 v0, v[29:32], off offset:128
; GFX11-NEXT: s_setpc_b64 s[30:31]
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) poison
%val = load { i32, <32 x i32> }, ptr addrspace(1) %ptr
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index fb0c03543377c..37491e9552207 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -152,8 +152,8 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_mov_b32 s16, 0
; CHECK-NEXT: s_mov_b32 s17, s16
; CHECK-NEXT: v_readlane_b32 s44, v7, 16
-; CHECK-NEXT: v_mov_b32_e32 v0, s16
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
+; CHECK-NEXT: v_mov_b32_e32 v2, s16
+; CHECK-NEXT: v_mov_b32_e32 v3, s17
; CHECK-NEXT: s_mov_b32 s18, s16
; CHECK-NEXT: s_mov_b32 s19, s16
; CHECK-NEXT: v_readlane_b32 s45, v7, 17
@@ -171,7 +171,7 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_readlane_b32 s57, v7, 29
; CHECK-NEXT: v_readlane_b32 s58, v7, 30
; CHECK-NEXT: v_readlane_b32 s59, v7, 31
-; CHECK-NEXT: image_sample_lz v2, v[0:1], s[44:51], s[16:19] dmask:0x1
+; CHECK-NEXT: image_sample_lz v1, v[2:3], s[44:51], s[16:19] dmask:0x1
; CHECK-NEXT: v_readlane_b32 s44, v7, 0
; CHECK-NEXT: v_readlane_b32 s52, v7, 8
; CHECK-NEXT: v_readlane_b32 s53, v7, 9
@@ -181,18 +181,18 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_readlane_b32 s57, v7, 13
; CHECK-NEXT: v_readlane_b32 s58, v7, 14
; CHECK-NEXT: v_readlane_b32 s59, v7, 15
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, v3
; CHECK-NEXT: v_readlane_b32 s45, v7, 1
; CHECK-NEXT: v_readlane_b32 s46, v7, 2
; CHECK-NEXT: v_readlane_b32 s47, v7, 3
-; CHECK-NEXT: image_sample_lz v0, v[0:1], s[52:59], s[24:27] dmask:0x1
; CHECK-NEXT: v_readlane_b32 s48, v7, 4
; CHECK-NEXT: v_readlane_b32 s49, v7, 5
+; CHECK-NEXT: image_sample_lz v0, v[2:3], s[52:59], s[24:27] dmask:0x1
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
; CHECK-NEXT: v_readlane_b32 s50, v7, 6
; CHECK-NEXT: v_readlane_b32 s51, v7, 7
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[16:19], 0
+; CHECK-NEXT: buffer_store_dwordx3 v[1:3], off, s[16:19], 0
; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; CHECK-NEXT: ; implicit-def: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll b/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
index 3b2d08eb26695..cb810d7d0bd3e 100644
--- a/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
@@ -532,11 +532,11 @@ define amdgpu_ps void @load_1d_v3f16_tfe_dmask7(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-NEXT: v_mov_b32_e32 v3, v1
; GFX8-UNPACKED-NEXT: v_mov_b32_e32 v4, v1
; GFX8-UNPACKED-NEXT: image_load v[1:4], v0, s[4:11] dmask:0x7 unorm tfe d16
-; GFX8-UNPACKED-NEXT: s_mov_b32 s0, 0x1000504
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-NEXT: v_perm_b32 v0, v1, v2, s0
+; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX8-UNPACKED-NEXT: flat_store_short v[0:1], v3
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: flat_store_dword v[0:1], v0
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
; GFX8-UNPACKED-NEXT: flat_store_dword v[0:1], v4
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 127f4996b7676..a1a11c4705056 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -1885,7 +1885,7 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
; NOOPT-NEXT: s_mov_b32 s21, 0x40400000
; NOOPT-NEXT: s_mov_b32 s22, 2.0
; NOOPT-NEXT: s_mov_b32 s23, 1.0
-; NOOPT-NEXT: v_mov_b32_e32 v7, s23
+; NOOPT-NEXT: v_mov_b32_e32 v0, s23
; NOOPT-NEXT: v_mov_b32_e32 v30, s22
; NOOPT-NEXT: v_mov_b32_e32 v29, s21
; NOOPT-NEXT: v_mov_b32_e32 v28, s20
@@ -1894,77 +1894,61 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
; NOOPT-NEXT: v_mov_b32_e32 v25, s17
; NOOPT-NEXT: v_mov_b32_e32 v24, s16
; NOOPT-NEXT: v_mov_b32_e32 v23, s15
-; NOOPT-NEXT: v_mov_b32_e32 v6, s14
-; NOOPT-NEXT: v_mov_b32_e32 v5, s13
-; NOOPT-NEXT: v_mov_b32_e32 v4, s12
-; NOOPT-NEXT: v_mov_b32_e32 v3, s11
-; NOOPT-NEXT: v_mov_b32_e32 v2, s10
-; NOOPT-NEXT: v_mov_b32_e32 v1, s9
-; NOOPT-NEXT: v_mov_b32_e32 v0, s8
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v8, v30
-; NOOPT-NEXT: v_mov_b32_e32 v9, v29
-; NOOPT-NEXT: v_mov_b32_e32 v10, v28
-; NOOPT-NEXT: v_mov_b32_e32 v11, v27
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v15, v23
-; NOOPT-NEXT: v_mov_b32_e32 v16, v6
-; NOOPT-NEXT: v_mov_b32_e32 v17, v5
-; NOOPT-NEXT: v_mov_b32_e32 v18, v4
-; NOOPT-NEXT: v_mov_b32_e32 v19, v3
-; NOOPT-NEXT: v_mov_b32_e32 v20, v2
-; NOOPT-NEXT: v_mov_b32_e32 v21, v1
-; NOOPT-NEXT: v_mov_b32_e32 v22, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x41880000
+; NOOPT-NEXT: v_mov_b32_e32 v22, s14
+; NOOPT-NEXT: v_mov_b32_e32 v21, s13
+; NOOPT-NEXT: v_mov_b32_e32 v20, s12
+; NOOPT-NEXT: v_mov_b32_e32 v19, s11
+; NOOPT-NEXT: v_mov_b32_e32 v18, s10
+; NOOPT-NEXT: v_mov_b32_e32 v17, s9
+; NOOPT-NEXT: v_mov_b32_e32 v16, s8
+; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v1, v30
+; NOOPT-NEXT: v_mov_b32_e32 v2, v29
+; NOOPT-NEXT: v_mov_b32_e32 v3, v28
+; NOOPT-NEXT: v_mov_b32_e32 v4, v27
+; NOOPT-NEXT: v_mov_b32_e32 v5, v26
+; NOOPT-NEXT: v_mov_b32_e32 v6, v25
+; NOOPT-NEXT: v_mov_b32_e32 v7, v24
+; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: v_mov_b32_e32 v9, v22
+; NOOPT-NEXT: v_mov_b32_e32 v10, v21
+; NOOPT-NEXT: v_mov_b32_e32 v11, v20
+; NOOPT-NEXT: v_mov_b32_e32 v12, v19
+; NOOPT-NEXT: v_mov_b32_e32 v13, v18
+; NOOPT-NEXT: v_mov_b32_e32 v14, v17
+; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x41880000
; NOOPT-NEXT: s_mov_b32 m0, s7
-; NOOPT-NEXT: v_movreld_b32_e32 v7, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v22
-; NOOPT-NEXT: v_mov_b32_e32 v5, v21
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -2209,7 +2193,8 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
; NOOPT-NEXT: s_mov_b32 s21, 0x40400000
; NOOPT-NEXT: s_mov_b32 s22, 2.0
; NOOPT-NEXT: s_mov_b32 s23, 1.0
-; NOOPT-NEXT: v_mov_b32_e32 v7, s23
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_mov_b32_e32 v0, s23
; NOOPT-NEXT: v_mov_b32_e32 v30, s22
; NOOPT-NEXT: v_mov_b32_e32 v29, s21
; NOOPT-NEXT: v_mov_b32_e32 v28, s20
@@ -2218,78 +2203,61 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
; NOOPT-NEXT: v_mov_b32_e32 v25, s17
; NOOPT-NEXT: v_mov_b32_e32 v24, s16
; NOOPT-NEXT: v_mov_b32_e32 v23, s15
-; NOOPT-NEXT: v_mov_b32_e32 v6, s14
-; NOOPT-NEXT: v_mov_b32_e32 v5, s13
-; NOOPT-NEXT: v_mov_b32_e32 v4, s12
-; NOOPT-NEXT: v_mov_b32_e32 v3, s11
-; NOOPT-NEXT: v_mov_b32_e32 v2, s10
-; NOOPT-NEXT: v_mov_b32_e32 v1, s9
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, s8
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v8, v30
-; NOOPT-NEXT: v_mov_b32_e32 v9, v29
-; NOOPT-NEXT: v_mov_b32_e32 v10, v28
-; NOOPT-NEXT: v_mov_b32_e32 v11, v27
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v15, v23
-; NOOPT-NEXT: v_mov_b32_e32 v16, v6
-; NOOPT-NEXT: v_mov_b32_e32 v17, v5
-; NOOPT-NEXT: v_mov_b32_e32 v18, v4
-; NOOPT-NEXT: v_mov_b32_e32 v19, v3
-; NOOPT-NEXT: v_mov_b32_e32 v20, v2
-; NOOPT-NEXT: v_mov_b32_e32 v21, v1
-; NOOPT-NEXT: v_mov_b32_e32 v22, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x41880000
+; NOOPT-NEXT: v_mov_b32_e32 v22, s14
+; NOOPT-NEXT: v_mov_b32_e32 v21, s13
+; NOOPT-NEXT: v_mov_b32_e32 v20, s12
+; NOOPT-NEXT: v_mov_b32_e32 v19, s11
+; NOOPT-NEXT: v_mov_b32_e32 v18, s10
+; NOOPT-NEXT: v_mov_b32_e32 v17, s9
+; NOOPT-NEXT: v_mov_b32_e32 v16, s8
+; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v1, v30
+; NOOPT-NEXT: v_mov_b32_e32 v2, v29
+; NOOPT-NEXT: v_mov_b32_e32 v3, v28
+; NOOPT-NEXT: v_mov_b32_e32 v4, v27
+; NOOPT-NEXT: v_mov_b32_e32 v5, v26
+; NOOPT-NEXT: v_mov_b32_e32 v6, v25
+; NOOPT-NEXT: v_mov_b32_e32 v7, v24
+; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: v_mov_b32_e32 v9, v22
+; NOOPT-NEXT: v_mov_b32_e32 v10, v21
+; NOOPT-NEXT: v_mov_b32_e32 v11, v20
+; NOOPT-NEXT: v_mov_b32_e32 v12, v19
+; NOOPT-NEXT: v_mov_b32_e32 v13, v18
+; NOOPT-NEXT: v_mov_b32_e32 v14, v17
+; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x41880000
; NOOPT-NEXT: s_mov_b32 m0, s7
-; NOOPT-NEXT: v_movreld_b32_e32 v8, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v22
-; NOOPT-NEXT: v_mov_b32_e32 v5, v21
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_movreld_b32_e32 v1, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -2537,7 +2505,8 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
; NOOPT-NEXT: s_mov_b32 s21, 0x40400000
; NOOPT-NEXT: s_mov_b32 s22, 2.0
; NOOPT-NEXT: s_mov_b32 s23, 1.0
-; NOOPT-NEXT: v_mov_b32_e32 v7, s23
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_mov_b32_e32 v0, s23
; NOOPT-NEXT: v_mov_b32_e32 v30, s22
; NOOPT-NEXT: v_mov_b32_e32 v29, s21
; NOOPT-NEXT: v_mov_b32_e32 v28, s20
@@ -2546,78 +2515,61 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
; NOOPT-NEXT: v_mov_b32_e32 v25, s17
; NOOPT-NEXT: v_mov_b32_e32 v24, s16
; NOOPT-NEXT: v_mov_b32_e32 v23, s15
-; NOOPT-NEXT: v_mov_b32_e32 v6, s14
-; NOOPT-NEXT: v_mov_b32_e32 v5, s13
-; NOOPT-NEXT: v_mov_b32_e32 v4, s12
-; NOOPT-NEXT: v_mov_b32_e32 v3, s11
-; NOOPT-NEXT: v_mov_b32_e32 v2, s10
-; NOOPT-NEXT: v_mov_b32_e32 v1, s9
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, s8
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v8, v30
-; NOOPT-NEXT: v_mov_b32_e32 v9, v29
-; NOOPT-NEXT: v_mov_b32_e32 v10, v28
-; NOOPT-NEXT: v_mov_b32_e32 v11, v27
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v15, v23
-; NOOPT-NEXT: v_mov_b32_e32 v16, v6
-; NOOPT-NEXT: v_mov_b32_e32 v17, v5
-; NOOPT-NEXT: v_mov_b32_e32 v18, v4
-; NOOPT-NEXT: v_mov_b32_e32 v19, v3
-; NOOPT-NEXT: v_mov_b32_e32 v20, v2
-; NOOPT-NEXT: v_mov_b32_e32 v21, v1
-; NOOPT-NEXT: v_mov_b32_e32 v22, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x41880000
+; NOOPT-NEXT: v_mov_b32_e32 v22, s14
+; NOOPT-NEXT: v_mov_b32_e32 v21, s13
+; NOOPT-NEXT: v_mov_b32_e32 v20, s12
+; NOOPT-NEXT: v_mov_b32_e32 v19, s11
+; NOOPT-NEXT: v_mov_b32_e32 v18, s10
+; NOOPT-NEXT: v_mov_b32_e32 v17, s9
+; NOOPT-NEXT: v_mov_b32_e32 v16, s8
+; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v1, v30
+; NOOPT-NEXT: v_mov_b32_e32 v2, v29
+; NOOPT-NEXT: v_mov_b32_e32 v3, v28
+; NOOPT-NEXT: v_mov_b32_e32 v4, v27
+; NOOPT-NEXT: v_mov_b32_e32 v5, v26
+; NOOPT-NEXT: v_mov_b32_e32 v6, v25
+; NOOPT-NEXT: v_mov_b32_e32 v7, v24
+; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: v_mov_b32_e32 v9, v22
+; NOOPT-NEXT: v_mov_b32_e32 v10, v21
+; NOOPT-NEXT: v_mov_b32_e32 v11, v20
+; NOOPT-NEXT: v_mov_b32_e32 v12, v19
+; NOOPT-NEXT: v_mov_b32_e32 v13, v18
+; NOOPT-NEXT: v_mov_b32_e32 v14, v17
+; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x41880000
; NOOPT-NEXT: s_mov_b32 m0, s7
-; NOOPT-NEXT: v_movreld_b32_e32 v7, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v22
-; NOOPT-NEXT: v_mov_b32_e32 v5, v21
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -2861,7 +2813,7 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
; NOOPT-NEXT: s_mov_b32 s21, 0x40400000
; NOOPT-NEXT: s_mov_b32 s22, 2.0
; NOOPT-NEXT: s_mov_b32 s23, 1.0
-; NOOPT-NEXT: v_mov_b32_e32 v7, s23
+; NOOPT-NEXT: v_mov_b32_e32 v0, s23
; NOOPT-NEXT: v_mov_b32_e32 v30, s22
; NOOPT-NEXT: v_mov_b32_e32 v29, s21
; NOOPT-NEXT: v_mov_b32_e32 v28, s20
@@ -2870,77 +2822,61 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
; NOOPT-NEXT: v_mov_b32_e32 v25, s17
; NOOPT-NEXT: v_mov_b32_e32 v24, s16
; NOOPT-NEXT: v_mov_b32_e32 v23, s15
-; NOOPT-NEXT: v_mov_b32_e32 v6, s14
-; NOOPT-NEXT: v_mov_b32_e32 v5, s13
-; NOOPT-NEXT: v_mov_b32_e32 v4, s12
-; NOOPT-NEXT: v_mov_b32_e32 v3, s11
-; NOOPT-NEXT: v_mov_b32_e32 v2, s10
-; NOOPT-NEXT: v_mov_b32_e32 v1, s9
-; NOOPT-NEXT: v_mov_b32_e32 v0, s8
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v8, v30
-; NOOPT-NEXT: v_mov_b32_e32 v9, v29
-; NOOPT-NEXT: v_mov_b32_e32 v10, v28
-; NOOPT-NEXT: v_mov_b32_e32 v11, v27
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v15, v23
-; NOOPT-NEXT: v_mov_b32_e32 v16, v6
-; NOOPT-NEXT: v_mov_b32_e32 v17, v5
-; NOOPT-NEXT: v_mov_b32_e32 v18, v4
-; NOOPT-NEXT: v_mov_b32_e32 v19, v3
-; NOOPT-NEXT: v_mov_b32_e32 v20, v2
-; NOOPT-NEXT: v_mov_b32_e32 v21, v1
-; NOOPT-NEXT: v_mov_b32_e32 v22, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x41880000
+; NOOPT-NEXT: v_mov_b32_e32 v22, s14
+; NOOPT-NEXT: v_mov_b32_e32 v21, s13
+; NOOPT-NEXT: v_mov_b32_e32 v20, s12
+; NOOPT-NEXT: v_mov_b32_e32 v19, s11
+; NOOPT-NEXT: v_mov_b32_e32 v18, s10
+; NOOPT-NEXT: v_mov_b32_e32 v17, s9
+; NOOPT-NEXT: v_mov_b32_e32 v16, s8
+; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v1, v30
+; NOOPT-NEXT: v_mov_b32_e32 v2, v29
+; NOOPT-NEXT: v_mov_b32_e32 v3, v28
+; NOOPT-NEXT: v_mov_b32_e32 v4, v27
+; NOOPT-NEXT: v_mov_b32_e32 v5, v26
+; NOOPT-NEXT: v_mov_b32_e32 v6, v25
+; NOOPT-NEXT: v_mov_b32_e32 v7, v24
+; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: v_mov_b32_e32 v9, v22
+; NOOPT-NEXT: v_mov_b32_e32 v10, v21
+; NOOPT-NEXT: v_mov_b32_e32 v11, v20
+; NOOPT-NEXT: v_mov_b32_e32 v12, v19
+; NOOPT-NEXT: v_mov_b32_e32 v13, v18
+; NOOPT-NEXT: v_mov_b32_e32 v14, v17
+; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x41880000
; NOOPT-NEXT: s_mov_b32 m0, s7
-; NOOPT-NEXT: v_movreld_b32_e32 v7, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v22
-; NOOPT-NEXT: v_mov_b32_e32 v5, v21
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -3175,83 +3111,69 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: s_mov_b32 s22, 2
; NOOPT-NEXT: s_mov_b32 s23, 1
; NOOPT-NEXT: s_mov_b32 s8, 0
-; NOOPT-NEXT: v_mov_b32_e32 v15, s8
-; NOOPT-NEXT: v_mov_b32_e32 v14, s23
-; NOOPT-NEXT: v_mov_b32_e32 v13, s22
-; NOOPT-NEXT: v_mov_b32_e32 v12, s21
-; NOOPT-NEXT: v_mov_b32_e32 v11, s20
-; NOOPT-NEXT: v_mov_b32_e32 v10, s19
-; NOOPT-NEXT: v_mov_b32_e32 v9, s18
-; NOOPT-NEXT: v_mov_b32_e32 v8, s17
-; NOOPT-NEXT: v_mov_b32_e32 v7, s16
-; NOOPT-NEXT: v_mov_b32_e32 v6, s15
-; NOOPT-NEXT: v_mov_b32_e32 v5, s14
-; NOOPT-NEXT: v_mov_b32_e32 v4, s13
-; NOOPT-NEXT: v_mov_b32_e32 v3, s12
-; NOOPT-NEXT: v_mov_b32_e32 v2, s11
-; NOOPT-NEXT: v_mov_b32_e32 v1, s10
-; NOOPT-NEXT: v_mov_b32_e32 v0, s9
-; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v16, v14
-; NOOPT-NEXT: v_mov_b32_e32 v17, v13
-; NOOPT-NEXT: v_mov_b32_e32 v18, v12
-; NOOPT-NEXT: v_mov_b32_e32 v19, v11
-; NOOPT-NEXT: v_mov_b32_e32 v20, v10
-; NOOPT-NEXT: v_mov_b32_e32 v21, v9
-; NOOPT-NEXT: v_mov_b32_e32 v22, v8
-; NOOPT-NEXT: v_mov_b32_e32 v23, v7
-; NOOPT-NEXT: v_mov_b32_e32 v24, v6
-; NOOPT-NEXT: v_mov_b32_e32 v25, v5
-; NOOPT-NEXT: v_mov_b32_e32 v26, v4
-; NOOPT-NEXT: v_mov_b32_e32 v27, v3
-; NOOPT-NEXT: v_mov_b32_e32 v28, v2
-; NOOPT-NEXT: v_mov_b32_e32 v29, v1
-; NOOPT-NEXT: v_mov_b32_e32 v30, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 16
-; NOOPT-NEXT: s_add_i32 m0, s7, 0xfffffe00
-; NOOPT-NEXT: v_movreld_b32_e32 v15, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: v_mov_b32_e32 v1, v22
-; NOOPT-NEXT: v_mov_b32_e32 v2, v21
-; NOOPT-NEXT: v_mov_b32_e32 v3, v20
-; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
+; NOOPT-NEXT: v_mov_b32_e32 v0, s8
+; NOOPT-NEXT: v_mov_b32_e32 v30, s23
+; NOOPT-NEXT: v_mov_b32_e32 v29, s22
+; NOOPT-NEXT: v_mov_b32_e32 v28, s21
+; NOOPT-NEXT: v_mov_b32_e32 v27, s20
+; NOOPT-NEXT: v_mov_b32_e32 v26, s19
+; NOOPT-NEXT: v_mov_b32_e32 v25, s18
+; NOOPT-NEXT: v_mov_b32_e32 v24, s17
+; NOOPT-NEXT: v_mov_b32_e32 v23, s16
+; NOOPT-NEXT: v_mov_b32_e32 v22, s15
+; NOOPT-NEXT: v_mov_b32_e32 v21, s14
+; NOOPT-NEXT: v_mov_b32_e32 v20, s13
+; NOOPT-NEXT: v_mov_b32_e32 v19, s12
+; NOOPT-NEXT: v_mov_b32_e32 v18, s11
+; NOOPT-NEXT: v_mov_b32_e32 v17, s10
+; NOOPT-NEXT: v_mov_b32_e32 v16, s9
+; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v1, v30
+; NOOPT-NEXT: v_mov_b32_e32 v2, v29
+; NOOPT-NEXT: v_mov_b32_e32 v3, v28
+; NOOPT-NEXT: v_mov_b32_e32 v4, v27
+; NOOPT-NEXT: v_mov_b32_e32 v5, v26
+; NOOPT-NEXT: v_mov_b32_e32 v6, v25
+; NOOPT-NEXT: v_mov_b32_e32 v7, v24
; NOOPT-NEXT: v_mov_b32_e32 v8, v23
-; NOOPT-NEXT: v_mov_b32_e32 v9, v30
-; NOOPT-NEXT: v_mov_b32_e32 v10, v29
-; NOOPT-NEXT: v_mov_b32_e32 v11, v28
-; NOOPT-NEXT: v_mov_b32_e32 v15, v27
-; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v16, v11
-; NOOPT-NEXT: v_mov_b32_e32 v17, v10
-; NOOPT-NEXT: v_mov_b32_e32 v18, v9
+; NOOPT-NEXT: v_mov_b32_e32 v9, v22
+; NOOPT-NEXT: v_mov_b32_e32 v10, v21
+; NOOPT-NEXT: v_mov_b32_e32 v11, v20
+; NOOPT-NEXT: v_mov_b32_e32 v12, v19
+; NOOPT-NEXT: v_mov_b32_e32 v13, v18
+; NOOPT-NEXT: v_mov_b32_e32 v14, v17
+; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, 16
+; NOOPT-NEXT: s_add_i32 m0, s7, 0xfffffe00
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v9, 32
-; NOOPT-NEXT: v_mov_b32_e32 v10, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v9, v14
-; NOOPT-NEXT: v_mov_b32_e32 v10, v13
-; NOOPT-NEXT: v_mov_b32_e32 v11, v12
-; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10 killed $exec
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v8, v3
-; NOOPT-NEXT: v_mov_b32_e32 v9, v2
-; NOOPT-NEXT: v_mov_b32_e32 v10, v1
-; NOOPT-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -3489,68 +3411,54 @@ define amdgpu_kernel void @insert_neg_offset_sgpr_loadreg(ptr addrspace(1) %in,
; NOOPT-NEXT: s_mov_b32 s1, s25
; NOOPT-NEXT: s_mov_b32 s2, s24
; NOOPT-NEXT: s_mov_b32 s3, s6
-; NOOPT-NEXT: v_mov_b32_e32 v0, 5
-; NOOPT-NEXT: v_mov_b32_e32 v30, s23
-; NOOPT-NEXT: v_mov_b32_e32 v29, s22
-; NOOPT-NEXT: v_mov_b32_e32 v28, s21
-; NOOPT-NEXT: v_mov_b32_e32 v27, s20
-; NOOPT-NEXT: v_mov_b32_e32 v26, s19
-; NOOPT-NEXT: v_mov_b32_e32 v25, s18
-; NOOPT-NEXT: v_mov_b32_e32 v24, s17
-; NOOPT-NEXT: v_mov_b32_e32 v23, s16
-; NOOPT-NEXT: v_mov_b32_e32 v22, s15
-; NOOPT-NEXT: v_mov_b32_e32 v21, s14
-; NOOPT-NEXT: v_mov_b32_e32 v20, s13
-; NOOPT-NEXT: v_mov_b32_e32 v19, s12
-; NOOPT-NEXT: v_mov_b32_e32 v18, s11
-; NOOPT-NEXT: v_mov_b32_e32 v17, s10
-; NOOPT-NEXT: v_mov_b32_e32 v16, s9
-; NOOPT-NEXT: v_mov_b32_e32 v15, s8
+; NOOPT-NEXT: v_mov_b32_e32 v16, 5
+; NOOPT-NEXT: v_mov_b32_e32 v0, s8
+; NOOPT-NEXT: v_mov_b32_e32 v1, s9
+; NOOPT-NEXT: v_mov_b32_e32 v2, s10
+; NOOPT-NEXT: v_mov_b32_e32 v3, s11
+; NOOPT-NEXT: v_mov_b32_e32 v4, s12
+; NOOPT-NEXT: v_mov_b32_e32 v5, s13
+; NOOPT-NEXT: v_mov_b32_e32 v6, s14
+; NOOPT-NEXT: v_mov_b32_e32 v7, s15
+; NOOPT-NEXT: v_mov_b32_e32 v8, s16
+; NOOPT-NEXT: v_mov_b32_e32 v9, s17
+; NOOPT-NEXT: v_mov_b32_e32 v10, s18
+; NOOPT-NEXT: v_mov_b32_e32 v11, s19
+; NOOPT-NEXT: v_mov_b32_e32 v12, s20
+; NOOPT-NEXT: v_mov_b32_e32 v13, s21
+; NOOPT-NEXT: v_mov_b32_e32 v14, s22
+; NOOPT-NEXT: v_mov_b32_e32 v15, s23
; NOOPT-NEXT: s_add_i32 m0, s7, 0xfffffe00
-; NOOPT-NEXT: v_movreld_b32_e32 v15, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: v_mov_b32_e32 v1, v22
-; NOOPT-NEXT: v_mov_b32_e32 v2, v21
-; NOOPT-NEXT: v_mov_b32_e32 v3, v20
-; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v8, v23
-; NOOPT-NEXT: v_mov_b32_e32 v9, v30
-; NOOPT-NEXT: v_mov_b32_e32 v10, v29
-; NOOPT-NEXT: v_mov_b32_e32 v11, v28
-; NOOPT-NEXT: v_mov_b32_e32 v15, v27
-; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v16, v11
-; NOOPT-NEXT: v_mov_b32_e32 v17, v10
-; NOOPT-NEXT: v_mov_b32_e32 v18, v9
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v9, 32
-; NOOPT-NEXT: v_mov_b32_e32 v10, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v9, v14
-; NOOPT-NEXT: v_mov_b32_e32 v10, v13
-; NOOPT-NEXT: v_mov_b32_e32 v11, v12
-; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10 killed $exec
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v8, v3
-; NOOPT-NEXT: v_mov_b32_e32 v9, v2
-; NOOPT-NEXT: v_mov_b32_e32 v10, v1
-; NOOPT-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -3987,69 +3895,52 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: v_readlane_b32 s4, v31, 0
; NOOPT-NEXT: v_readlane_b32 s5, v31, 1
; NOOPT-NEXT: v_readlane_b32 s6, v31, 2
-; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
-; NOOPT-NEXT: v_mov_b32_e32 v1, v22
-; NOOPT-NEXT: v_mov_b32_e32 v2, v21
-; NOOPT-NEXT: v_mov_b32_e32 v3, v20
-; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v3, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v4, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v5, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v6, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v7, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v8, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v9, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v10, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v11, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v12, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v13, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v14, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v9, v30
-; NOOPT-NEXT: v_mov_b32_e32 v10, v29
-; NOOPT-NEXT: v_mov_b32_e32 v11, v28
-; NOOPT-NEXT: v_mov_b32_e32 v15, v27
-; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v16, v11
-; NOOPT-NEXT: v_mov_b32_e32 v17, v10
-; NOOPT-NEXT: v_mov_b32_e32 v18, v9
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v9, 32
-; NOOPT-NEXT: v_mov_b32_e32 v10, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v9, v14
-; NOOPT-NEXT: v_mov_b32_e32 v10, v13
-; NOOPT-NEXT: v_mov_b32_e32 v11, v12
-; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10 killed $exec
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v8, v3
-; NOOPT-NEXT: v_mov_b32_e32 v9, v2
-; NOOPT-NEXT: v_mov_b32_e32 v10, v1
-; NOOPT-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -4476,69 +4367,52 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: v_readlane_b32 s4, v31, 0
; NOOPT-NEXT: v_readlane_b32 s5, v31, 1
; NOOPT-NEXT: v_readlane_b32 s6, v31, 2
-; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
-; NOOPT-NEXT: v_mov_b32_e32 v1, v22
-; NOOPT-NEXT: v_mov_b32_e32 v2, v21
-; NOOPT-NEXT: v_mov_b32_e32 v3, v20
-; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
-; NOOPT-NEXT: v_mov_b32_e32 v12, v26
-; NOOPT-NEXT: v_mov_b32_e32 v13, v25
-; NOOPT-NEXT: v_mov_b32_e32 v14, v24
-; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v3, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v4, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v5, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v6, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v7, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v8, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v9, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v10, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v11, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v12, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v13, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v14, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v9, v30
-; NOOPT-NEXT: v_mov_b32_e32 v10, v29
-; NOOPT-NEXT: v_mov_b32_e32 v11, v28
-; NOOPT-NEXT: v_mov_b32_e32 v15, v27
-; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v16, v11
-; NOOPT-NEXT: v_mov_b32_e32 v17, v10
-; NOOPT-NEXT: v_mov_b32_e32 v18, v9
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v9, 32
-; NOOPT-NEXT: v_mov_b32_e32 v10, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v9, v14
-; NOOPT-NEXT: v_mov_b32_e32 v10, v13
-; NOOPT-NEXT: v_mov_b32_e32 v11, v12
-; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10 killed $exec
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v8, v3
-; NOOPT-NEXT: v_mov_b32_e32 v9, v2
-; NOOPT-NEXT: v_mov_b32_e32 v10, v1
-; NOOPT-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -5581,10 +5455,10 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x19
; NOOPT-NEXT: s_waitcnt lgkmcnt(0)
; NOOPT-NEXT: s_load_dwordx2 s[18:19], s[4:5], 0x9
-; NOOPT-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane
+; NOOPT-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
; NOOPT-NEXT: s_waitcnt lgkmcnt(0)
-; NOOPT-NEXT: v_writelane_b32 v32, s18, 0
-; NOOPT-NEXT: v_writelane_b32 v32, s19, 1
+; NOOPT-NEXT: v_writelane_b32 v31, s18, 0
+; NOOPT-NEXT: v_writelane_b32 v31, s19, 1
; NOOPT-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0xd
; NOOPT-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x19
; NOOPT-NEXT: s_mov_b32 s24, s19
@@ -5595,10 +5469,10 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_mov_b32 s21, s24
; NOOPT-NEXT: s_mov_b32 s22, s19
; NOOPT-NEXT: s_mov_b32 s23, s18
-; NOOPT-NEXT: v_writelane_b32 v32, s20, 2
-; NOOPT-NEXT: v_writelane_b32 v32, s21, 3
-; NOOPT-NEXT: v_writelane_b32 v32, s22, 4
-; NOOPT-NEXT: v_writelane_b32 v32, s23, 5
+; NOOPT-NEXT: v_writelane_b32 v31, s20, 2
+; NOOPT-NEXT: v_writelane_b32 v31, s21, 3
+; NOOPT-NEXT: v_writelane_b32 v31, s22, 4
+; NOOPT-NEXT: v_writelane_b32 v31, s23, 5
; NOOPT-NEXT: v_mov_b32_e32 v2, 0
; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v1, v2
@@ -5606,11 +5480,11 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: v_lshl_b64 v[0:1], v[0:1], s19
; NOOPT-NEXT: s_mov_b32 s20, 0
-; NOOPT-NEXT: v_writelane_b32 v32, s20, 6
+; NOOPT-NEXT: v_writelane_b32 v31, s20, 6
; NOOPT-NEXT: ; kill: def $sgpr20 killed $sgpr20 def $sgpr20_sgpr21
; NOOPT-NEXT: s_mov_b32 s21, s18
-; NOOPT-NEXT: v_writelane_b32 v32, s20, 7
-; NOOPT-NEXT: v_writelane_b32 v32, s21, 8
+; NOOPT-NEXT: v_writelane_b32 v31, s20, 7
+; NOOPT-NEXT: v_writelane_b32 v31, s21, 8
; NOOPT-NEXT: ; kill: def $sgpr16_sgpr17 killed $sgpr16_sgpr17 def $sgpr16_sgpr17_sgpr18_sgpr19
; NOOPT-NEXT: s_mov_b64 s[18:19], s[20:21]
; NOOPT-NEXT: s_waitcnt lgkmcnt(0)
@@ -5645,10 +5519,10 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: v_mov_b32_e32 v14, s14
; NOOPT-NEXT: v_mov_b32_e32 v15, s15
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
-; NOOPT-NEXT: v_writelane_b32 v32, s0, 9
-; NOOPT-NEXT: v_writelane_b32 v32, s1, 10
+; NOOPT-NEXT: v_writelane_b32 v31, s0, 9
+; NOOPT-NEXT: v_writelane_b32 v31, s1, 10
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_store_dword v32, off, s[28:31], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v31, off, s[28:31], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: buffer_store_dword v0, off, s[28:31], 0 offset:4 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[28:31], 0 offset:8 ; 4-byte Folded Spill
@@ -5670,11 +5544,11 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 11
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 12
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 11
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 12
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:12 ; 4-byte Folded Reload
@@ -5739,25 +5613,25 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_store_dword v14, off, s[28:31], 0 offset:60 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[28:31], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
-; NOOPT-NEXT: v_writelane_b32 v32, s2, 11
-; NOOPT-NEXT: v_writelane_b32 v32, s3, 12
+; NOOPT-NEXT: v_writelane_b32 v31, s2, 11
+; NOOPT-NEXT: v_writelane_b32 v31, s3, 12
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_store_dword v32, off, s[28:31], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v31, off, s[28:31], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_xor_b64 exec, exec, s[0:1]
; NOOPT-NEXT: s_cbranch_execnz .LBB17_1
; NOOPT-NEXT: ; %bb.2:
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 9
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 10
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 9
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 10
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:88 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:92 ; 4-byte Folded Reload
@@ -5779,10 +5653,10 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_store_dword v16, off, s[28:31], 0 offset:216 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
; NOOPT-NEXT: s_waitcnt vmcnt(14)
-; NOOPT-NEXT: v_writelane_b32 v32, s0, 13
-; NOOPT-NEXT: v_writelane_b32 v32, s1, 14
+; NOOPT-NEXT: v_writelane_b32 v31, s0, 13
+; NOOPT-NEXT: v_writelane_b32 v31, s1, 14
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_store_dword v32, off, s[28:31], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v31, off, s[28:31], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: buffer_store_dword v0, off, s[28:31], 0 offset:152 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[28:31], 0 offset:156 ; 4-byte Folded Spill
@@ -5808,11 +5682,11 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: .LBB17_4: ; =>This Inner Loop Header: Depth=1
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 15
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 16
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 15
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 16
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:152 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:160 ; 4-byte Folded Reload
@@ -5877,107 +5751,90 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_store_dword v14, off, s[28:31], 0 offset:208 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[28:31], 0 offset:212 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
-; NOOPT-NEXT: v_writelane_b32 v32, s2, 15
-; NOOPT-NEXT: v_writelane_b32 v32, s3, 16
+; NOOPT-NEXT: v_writelane_b32 v31, s2, 15
+; NOOPT-NEXT: v_writelane_b32 v31, s3, 16
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_store_dword v32, off, s[28:31], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v31, off, s[28:31], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_xor_b64 exec, exec, s[0:1]
; NOOPT-NEXT: s_cbranch_execnz .LBB17_4
; NOOPT-NEXT: ; %bb.5:
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 13
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 14
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 13
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 14
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.6:
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 6
-; NOOPT-NEXT: v_readlane_b32 s4, v32, 2
-; NOOPT-NEXT: v_readlane_b32 s5, v32, 3
-; NOOPT-NEXT: v_readlane_b32 s6, v32, 4
-; NOOPT-NEXT: v_readlane_b32 s7, v32, 5
-; NOOPT-NEXT: v_readlane_b32 s2, v32, 7
-; NOOPT-NEXT: v_readlane_b32 s3, v32, 8
-; NOOPT-NEXT: v_readlane_b32 s8, v32, 0
-; NOOPT-NEXT: v_readlane_b32 s9, v32, 1
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
+; NOOPT-NEXT: v_readlane_b32 s4, v31, 2
+; NOOPT-NEXT: v_readlane_b32 s5, v31, 3
+; NOOPT-NEXT: v_readlane_b32 s6, v31, 4
+; NOOPT-NEXT: v_readlane_b32 s7, v31, 5
+; NOOPT-NEXT: v_readlane_b32 s2, v31, 7
+; NOOPT-NEXT: v_readlane_b32 s3, v31, 8
+; NOOPT-NEXT: v_readlane_b32 s8, v31, 0
+; NOOPT-NEXT: v_readlane_b32 s9, v31, 1
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:84 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:220 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:224 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[28:31], 0 offset:228 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[28:31], 0 offset:232 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[28:31], 0 offset:236 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[28:31], 0 offset:240 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[28:31], 0 offset:244 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[28:31], 0 offset:248 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[28:31], 0 offset:252 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[28:31], 0 offset:256 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[28:31], 0 offset:260 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[28:31], 0 offset:264 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[28:31], 0 offset:268 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[28:31], 0 offset:272 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[28:31], 0 offset:276 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 offset:280 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
-; NOOPT-NEXT: v_mov_b32_e32 v5, v19
-; NOOPT-NEXT: v_mov_b32_e32 v6, v18
-; NOOPT-NEXT: v_mov_b32_e32 v7, v17
-; NOOPT-NEXT: v_mov_b32_e32 v1, v16
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v22
-; NOOPT-NEXT: v_mov_b32_e32 v4, v21
-; NOOPT-NEXT: v_mov_b32_e32 v8, v20
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
-; NOOPT-NEXT: v_mov_b32_e32 v13, v27
-; NOOPT-NEXT: v_mov_b32_e32 v14, v26
-; NOOPT-NEXT: v_mov_b32_e32 v15, v25
-; NOOPT-NEXT: v_mov_b32_e32 v9, v24
+; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:220 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:224 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v3, off, s[28:31], 0 offset:228 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v4, off, s[28:31], 0 offset:232 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v5, off, s[28:31], 0 offset:236 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v6, off, s[28:31], 0 offset:240 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v7, off, s[28:31], 0 offset:244 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v8, off, s[28:31], 0 offset:248 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v9, off, s[28:31], 0 offset:252 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v10, off, s[28:31], 0 offset:256 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v11, off, s[28:31], 0 offset:260 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v12, off, s[28:31], 0 offset:264 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v13, off, s[28:31], 0 offset:268 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v14, off, s[28:31], 0 offset:272 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:276 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:280 ; 4-byte Folded Reload
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v10, v31
-; NOOPT-NEXT: v_mov_b32_e32 v11, v30
-; NOOPT-NEXT: v_mov_b32_e32 v12, v29
-; NOOPT-NEXT: v_mov_b32_e32 v16, v28
-; NOOPT-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17_vgpr18_vgpr19 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v17, v12
-; NOOPT-NEXT: v_mov_b32_e32 v18, v11
-; NOOPT-NEXT: v_mov_b32_e32 v19, v10
+; NOOPT-NEXT: v_mov_b32_e32 v24, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v28, v24
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
; NOOPT-NEXT: ; kill: def $sgpr8_sgpr9 killed $sgpr8_sgpr9 def $sgpr8_sgpr9_sgpr10_sgpr11
; NOOPT-NEXT: s_mov_b64 s[10:11], s[2:3]
-; NOOPT-NEXT: v_mov_b32_e32 v10, 32
-; NOOPT-NEXT: v_mov_b32_e32 v11, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], v[10:11], s[8:11], 0 addr64 offset:16
+; NOOPT-NEXT: v_mov_b32_e32 v29, 32
+; NOOPT-NEXT: v_mov_b32_e32 v30, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[25:28], v[29:30], s[8:11], 0 addr64 offset:16
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10_vgpr11_vgpr12 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v10, v15
-; NOOPT-NEXT: v_mov_b32_e32 v11, v14
-; NOOPT-NEXT: v_mov_b32_e32 v12, v13
-; NOOPT-NEXT: buffer_store_dwordx4 v[9:12], off, s[4:7], 0 offset:32
-; NOOPT-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; NOOPT-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v9, v4
-; NOOPT-NEXT: v_mov_b32_e32 v10, v3
-; NOOPT-NEXT: v_mov_b32_e32 v11, v2
-; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr17_vgpr18_vgpr19_vgpr20 killed $vgpr17_vgpr18_vgpr19_vgpr20 killed $vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[17:20], off, s[4:7], 0 offset:32
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v2, v7
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
-; NOOPT-NEXT: v_mov_b32_e32 v4, v5
+; NOOPT-NEXT: v_mov_b32_e32 v12, v8
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: buffer_store_dwordx4 v[9:12], off, s[4:7], 0 offset:16
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: ; kill: def $vgpr1_vgpr2_vgpr3_vgpr4 killed $vgpr1_vgpr2_vgpr3_vgpr4 killed $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[1:4], off, s[4:7], 0
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
-; NOOPT-NEXT: v_writelane_b32 v32, s0, 17
-; NOOPT-NEXT: v_writelane_b32 v32, s1, 18
+; NOOPT-NEXT: v_writelane_b32 v31, s0, 17
+; NOOPT-NEXT: v_writelane_b32 v31, s1, 18
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_store_dword v32, off, s[28:31], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v31, off, s[28:31], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
@@ -6000,11 +5857,11 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: .LBB17_8: ; %bb2
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 17
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 18
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 17
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 18
; NOOPT-NEXT: s_or_b64 exec, exec, s[0:1]
; NOOPT-NEXT: s_endpgm
;
@@ -6476,7 +6333,7 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
; NOOPT-NEXT: s_mov_b32 s22, 0x40400000
; NOOPT-NEXT: s_mov_b32 s23, 2.0
; NOOPT-NEXT: s_mov_b32 s24, 1.0
-; NOOPT-NEXT: v_mov_b32_e32 v25, s24
+; NOOPT-NEXT: v_mov_b32_e32 v30, s24
; NOOPT-NEXT: v_mov_b32_e32 v14, s23
; NOOPT-NEXT: v_mov_b32_e32 v13, s22
; NOOPT-NEXT: v_mov_b32_e32 v12, s21
@@ -6492,91 +6349,83 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
; NOOPT-NEXT: v_mov_b32_e32 v2, s9
; NOOPT-NEXT: v_mov_b32_e32 v1, s8
; NOOPT-NEXT: v_mov_b32_e32 v0, s7
-; NOOPT-NEXT: ; kill: def $vgpr25 killed $vgpr25 def $vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v26, v14
-; NOOPT-NEXT: v_mov_b32_e32 v27, v13
-; NOOPT-NEXT: v_mov_b32_e32 v28, v12
-; NOOPT-NEXT: v_mov_b32_e32 v29, v11
-; NOOPT-NEXT: v_mov_b32_e32 v30, v10
-; NOOPT-NEXT: v_mov_b32_e32 v31, v9
-; NOOPT-NEXT: v_mov_b32_e32 v32, v8
-; NOOPT-NEXT: v_mov_b32_e32 v33, v7
-; NOOPT-NEXT: v_mov_b32_e32 v34, v6
-; NOOPT-NEXT: v_mov_b32_e32 v35, v5
-; NOOPT-NEXT: v_mov_b32_e32 v36, v4
-; NOOPT-NEXT: v_mov_b32_e32 v37, v3
-; NOOPT-NEXT: v_mov_b32_e32 v38, v2
-; NOOPT-NEXT: v_mov_b32_e32 v39, v1
-; NOOPT-NEXT: v_mov_b32_e32 v40, v0
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x41880000
+; NOOPT-NEXT: ; kill: def $vgpr30 killed $vgpr30 def $vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v31, v14
+; NOOPT-NEXT: v_mov_b32_e32 v32, v13
+; NOOPT-NEXT: v_mov_b32_e32 v33, v12
+; NOOPT-NEXT: v_mov_b32_e32 v34, v11
+; NOOPT-NEXT: v_mov_b32_e32 v35, v10
+; NOOPT-NEXT: v_mov_b32_e32 v36, v9
+; NOOPT-NEXT: v_mov_b32_e32 v37, v8
+; NOOPT-NEXT: v_mov_b32_e32 v38, v7
+; NOOPT-NEXT: v_mov_b32_e32 v39, v6
+; NOOPT-NEXT: v_mov_b32_e32 v40, v5
+; NOOPT-NEXT: v_mov_b32_e32 v41, v4
+; NOOPT-NEXT: v_mov_b32_e32 v42, v3
+; NOOPT-NEXT: v_mov_b32_e32 v43, v2
+; NOOPT-NEXT: v_mov_b32_e32 v44, v1
+; NOOPT-NEXT: v_mov_b32_e32 v45, v0
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x41880000
; NOOPT-NEXT: s_mov_b32 m0, s6
-; NOOPT-NEXT: v_movreld_b32_e32 v25, v0
+; NOOPT-NEXT: v_movreld_b32_e32 v30, v16
; NOOPT-NEXT: s_mov_b32 s6, 2
; NOOPT-NEXT: s_add_i32 s5, s5, s6
; NOOPT-NEXT: s_mov_b32 m0, s5
-; NOOPT-NEXT: v_mov_b32_e32 v7, v25
-; NOOPT-NEXT: v_mov_b32_e32 v8, v26
-; NOOPT-NEXT: v_mov_b32_e32 v9, v27
-; NOOPT-NEXT: v_mov_b32_e32 v10, v28
-; NOOPT-NEXT: v_mov_b32_e32 v11, v29
-; NOOPT-NEXT: v_mov_b32_e32 v12, v30
-; NOOPT-NEXT: v_mov_b32_e32 v13, v31
-; NOOPT-NEXT: v_mov_b32_e32 v14, v32
-; NOOPT-NEXT: v_mov_b32_e32 v15, v33
-; NOOPT-NEXT: v_mov_b32_e32 v16, v34
-; NOOPT-NEXT: v_mov_b32_e32 v17, v35
-; NOOPT-NEXT: v_mov_b32_e32 v18, v36
-; NOOPT-NEXT: v_mov_b32_e32 v19, v37
-; NOOPT-NEXT: v_mov_b32_e32 v20, v38
-; NOOPT-NEXT: v_mov_b32_e32 v21, v39
-; NOOPT-NEXT: v_mov_b32_e32 v22, v40
-; NOOPT-NEXT: v_movreld_b32_e32 v7, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v40
-; NOOPT-NEXT: v_mov_b32_e32 v5, v39
-; NOOPT-NEXT: v_mov_b32_e32 v6, v38
-; NOOPT-NEXT: v_mov_b32_e32 v0, v37
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v0, v30
+; NOOPT-NEXT: v_mov_b32_e32 v1, v31
+; NOOPT-NEXT: v_mov_b32_e32 v2, v32
+; NOOPT-NEXT: v_mov_b32_e32 v3, v33
+; NOOPT-NEXT: v_mov_b32_e32 v4, v34
+; NOOPT-NEXT: v_mov_b32_e32 v5, v35
+; NOOPT-NEXT: v_mov_b32_e32 v6, v36
+; NOOPT-NEXT: v_mov_b32_e32 v7, v37
+; NOOPT-NEXT: v_mov_b32_e32 v8, v38
+; NOOPT-NEXT: v_mov_b32_e32 v9, v39
+; NOOPT-NEXT: v_mov_b32_e32 v10, v40
+; NOOPT-NEXT: v_mov_b32_e32 v11, v41
+; NOOPT-NEXT: v_mov_b32_e32 v12, v42
+; NOOPT-NEXT: v_mov_b32_e32 v13, v43
+; NOOPT-NEXT: v_mov_b32_e32 v14, v44
+; NOOPT-NEXT: v_mov_b32_e32 v15, v45
+; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
+; NOOPT-NEXT: v_mov_b32_e32 v16, v38
+; NOOPT-NEXT: v_mov_b32_e32 v17, v39
+; NOOPT-NEXT: v_mov_b32_e32 v18, v40
+; NOOPT-NEXT: v_mov_b32_e32 v19, v41
+; NOOPT-NEXT: v_mov_b32_e32 v20, v42
+; NOOPT-NEXT: v_mov_b32_e32 v21, v43
+; NOOPT-NEXT: v_mov_b32_e32 v22, v44
+; NOOPT-NEXT: v_mov_b32_e32 v23, v45
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: s_mov_b32 s10, 0
; NOOPT-NEXT: ; kill: def $sgpr10 killed $sgpr10 def $sgpr10_sgpr11
; NOOPT-NEXT: s_mov_b32 s11, s4
; NOOPT-NEXT: s_mov_b64 s[4:5], s[14:15]
; NOOPT-NEXT: s_mov_b64 s[6:7], s[10:11]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v6, v36
-; NOOPT-NEXT: v_mov_b32_e32 v23, v35
-; NOOPT-NEXT: v_mov_b32_e32 v24, v34
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v33
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v24
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v6, v32
-; NOOPT-NEXT: v_mov_b32_e32 v23, v31
-; NOOPT-NEXT: v_mov_b32_e32 v24, v30
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v29
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v24
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v6, v28
-; NOOPT-NEXT: v_mov_b32_e32 v23, v27
-; NOOPT-NEXT: v_mov_b32_e32 v24, v26
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v25
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v24
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; NOOPT-NEXT: v_mov_b32_e32 v16, v30
+; NOOPT-NEXT: v_mov_b32_e32 v17, v31
+; NOOPT-NEXT: v_mov_b32_e32 v18, v32
+; NOOPT-NEXT: v_mov_b32_e32 v19, v33
+; NOOPT-NEXT: v_mov_b32_e32 v20, v34
+; NOOPT-NEXT: v_mov_b32_e32 v21, v35
+; NOOPT-NEXT: v_mov_b32_e32 v22, v36
+; NOOPT-NEXT: v_mov_b32_e32 v23, v37
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0
; NOOPT-NEXT: s_mov_b64 s[8:9], 64
; NOOPT-NEXT: s_mov_b32 s12, s14
; NOOPT-NEXT: s_mov_b32 s13, s15
@@ -6586,51 +6435,35 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
; NOOPT-NEXT: s_addc_u32 s14, s13, s14
; NOOPT-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13
; NOOPT-NEXT: s_mov_b32 s13, s14
-; NOOPT-NEXT: v_mov_b32_e32 v6, v22
-; NOOPT-NEXT: v_mov_b32_e32 v23, v21
-; NOOPT-NEXT: v_mov_b32_e32 v24, v20
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v24
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
; NOOPT-NEXT: ; kill: def $sgpr12_sgpr13 killed $sgpr12_sgpr13 def $sgpr12_sgpr13_sgpr14_sgpr15
; NOOPT-NEXT: s_mov_b64 s[14:15], s[10:11]
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[12:15], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: v_mov_b32_e32 v4, s8
-; NOOPT-NEXT: v_mov_b32_e32 v5, s9
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: v_mov_b32_e32 v4, s8
-; NOOPT-NEXT: v_mov_b32_e32 v5, s9
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[12:15], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v21, s9
+; NOOPT-NEXT: v_mov_b32_e32 v20, s8
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], v[20:21], s[4:7], 0 addr64 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: v_mov_b32_e32 v13, s9
+; NOOPT-NEXT: v_mov_b32_e32 v12, s8
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], v[12:13], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; NOOPT-NEXT: s_endpgm
;
@@ -8475,70 +8308,54 @@ define amdgpu_kernel void @insertelement_v16f32_or_index(ptr addrspace(1) %out,
; NOOPT-NEXT: s_mov_b32 s3, s6
; NOOPT-NEXT: s_mov_b32 s24, 2
; NOOPT-NEXT: s_lshl_b32 s7, s7, s24
-; NOOPT-NEXT: v_mov_b32_e32 v0, 0x40a00000
-; NOOPT-NEXT: v_mov_b32_e32 v7, s8
-; NOOPT-NEXT: v_mov_b32_e32 v8, s9
-; NOOPT-NEXT: v_mov_b32_e32 v9, s10
-; NOOPT-NEXT: v_mov_b32_e32 v10, s11
-; NOOPT-NEXT: v_mov_b32_e32 v11, s12
-; NOOPT-NEXT: v_mov_b32_e32 v12, s13
-; NOOPT-NEXT: v_mov_b32_e32 v13, s14
-; NOOPT-NEXT: v_mov_b32_e32 v14, s15
-; NOOPT-NEXT: v_mov_b32_e32 v15, s16
-; NOOPT-NEXT: v_mov_b32_e32 v16, s17
-; NOOPT-NEXT: v_mov_b32_e32 v17, s18
-; NOOPT-NEXT: v_mov_b32_e32 v18, s19
-; NOOPT-NEXT: v_mov_b32_e32 v19, s20
-; NOOPT-NEXT: v_mov_b32_e32 v20, s21
-; NOOPT-NEXT: v_mov_b32_e32 v21, s22
-; NOOPT-NEXT: v_mov_b32_e32 v22, s23
+; NOOPT-NEXT: v_mov_b32_e32 v16, 0x40a00000
+; NOOPT-NEXT: v_mov_b32_e32 v0, s8
+; NOOPT-NEXT: v_mov_b32_e32 v1, s9
+; NOOPT-NEXT: v_mov_b32_e32 v2, s10
+; NOOPT-NEXT: v_mov_b32_e32 v3, s11
+; NOOPT-NEXT: v_mov_b32_e32 v4, s12
+; NOOPT-NEXT: v_mov_b32_e32 v5, s13
+; NOOPT-NEXT: v_mov_b32_e32 v6, s14
+; NOOPT-NEXT: v_mov_b32_e32 v7, s15
+; NOOPT-NEXT: v_mov_b32_e32 v8, s16
+; NOOPT-NEXT: v_mov_b32_e32 v9, s17
+; NOOPT-NEXT: v_mov_b32_e32 v10, s18
+; NOOPT-NEXT: v_mov_b32_e32 v11, s19
+; NOOPT-NEXT: v_mov_b32_e32 v12, s20
+; NOOPT-NEXT: v_mov_b32_e32 v13, s21
+; NOOPT-NEXT: v_mov_b32_e32 v14, s22
+; NOOPT-NEXT: v_mov_b32_e32 v15, s23
; NOOPT-NEXT: s_mov_b32 m0, s7
-; NOOPT-NEXT: v_movreld_b32_e32 v8, v0
-; NOOPT-NEXT: v_mov_b32_e32 v4, v22
-; NOOPT-NEXT: v_mov_b32_e32 v5, v21
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v0, v19
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_movreld_b32_e32 v1, v16
; NOOPT-NEXT: s_mov_b32 s8, 0
; NOOPT-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
; NOOPT-NEXT: s_mov_b32 s9, s6
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v4, 32
-; NOOPT-NEXT: v_mov_b32_e32 v5, 0
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v18
-; NOOPT-NEXT: v_mov_b32_e32 v5, v17
-; NOOPT-NEXT: v_mov_b32_e32 v6, v16
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; NOOPT-NEXT: v_mov_b32_e32 v4, v14
-; NOOPT-NEXT: v_mov_b32_e32 v5, v13
-; NOOPT-NEXT: v_mov_b32_e32 v6, v12
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v11
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; NOOPT-NEXT: v_mov_b32_e32 v4, v10
-; NOOPT-NEXT: v_mov_b32_e32 v5, v9
-; NOOPT-NEXT: v_mov_b32_e32 v6, v8
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v0, v7
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v6
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: v_mov_b32_e32 v3, v4
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v21, v13
+; NOOPT-NEXT: v_mov_b32_e32 v20, v12
+; NOOPT-NEXT: v_mov_b32_e32 v19, v11
+; NOOPT-NEXT: v_mov_b32_e32 v18, v10
+; NOOPT-NEXT: v_mov_b32_e32 v17, v9
+; NOOPT-NEXT: v_mov_b32_e32 v16, v8
+; NOOPT-NEXT: v_mov_b32_e32 v27, v23
+; NOOPT-NEXT: v_mov_b32_e32 v26, v22
+; NOOPT-NEXT: v_mov_b32_e32 v25, v21
+; NOOPT-NEXT: v_mov_b32_e32 v24, v20
+; NOOPT-NEXT: v_mov_b32_e32 v28, 32
+; NOOPT-NEXT: v_mov_b32_e32 v29, 0
+; NOOPT-NEXT: buffer_store_dwordx4 v[24:27], v[28:29], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19 killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v11, v7
+; NOOPT-NEXT: v_mov_b32_e32 v10, v6
+; NOOPT-NEXT: v_mov_b32_e32 v9, v5
+; NOOPT-NEXT: v_mov_b32_e32 v8, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -9253,13 +9070,13 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_mov_b32 s19, 0xe8f000
; NOOPT-NEXT: s_add_u32 s16, s16, s5
; NOOPT-NEXT: s_addc_u32 s17, s17, 0
-; NOOPT-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; NOOPT-NEXT: v_writelane_b32 v33, s4, 0
+; NOOPT-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane
+; NOOPT-NEXT: v_writelane_b32 v32, s4, 0
; NOOPT-NEXT: s_mov_b32 s4, s1
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 0
-; NOOPT-NEXT: v_writelane_b32 v33, s4, 1
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 0
+; NOOPT-NEXT: v_writelane_b32 v32, s4, 1
; NOOPT-NEXT: s_mov_b32 s4, s0
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 1
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 1
; NOOPT-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:144 ; 4-byte Folded Spill
; NOOPT-NEXT: v_mov_b32_e32 v2, v1
; NOOPT-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
@@ -9268,19 +9085,19 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_mov_b32 s7, s3
; NOOPT-NEXT: s_mov_b32 s8, 0xf000
; NOOPT-NEXT: s_mov_b32 s0, 0
-; NOOPT-NEXT: v_writelane_b32 v33, s0, 2
+; NOOPT-NEXT: v_writelane_b32 v32, s0, 2
; NOOPT-NEXT: s_mov_b32 s2, s0
; NOOPT-NEXT: s_mov_b32 s3, s8
-; NOOPT-NEXT: v_writelane_b32 v33, s2, 3
-; NOOPT-NEXT: v_writelane_b32 v33, s3, 4
+; NOOPT-NEXT: v_writelane_b32 v32, s2, 3
+; NOOPT-NEXT: v_writelane_b32 v32, s3, 4
; NOOPT-NEXT: s_mov_b32 s8, s0
; NOOPT-NEXT: s_mov_b32 s9, s0
; NOOPT-NEXT: ; kill: def $sgpr8_sgpr9 killed $sgpr8_sgpr9 def $sgpr8_sgpr9_sgpr10_sgpr11
; NOOPT-NEXT: s_mov_b64 s[10:11], s[2:3]
-; NOOPT-NEXT: v_writelane_b32 v33, s8, 5
-; NOOPT-NEXT: v_writelane_b32 v33, s9, 6
-; NOOPT-NEXT: v_writelane_b32 v33, s10, 7
-; NOOPT-NEXT: v_writelane_b32 v33, s11, 8
+; NOOPT-NEXT: v_writelane_b32 v32, s8, 5
+; NOOPT-NEXT: v_writelane_b32 v32, s9, 6
+; NOOPT-NEXT: v_writelane_b32 v32, s10, 7
+; NOOPT-NEXT: v_writelane_b32 v32, s11, 8
; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v1, v2
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:136 ; 4-byte Folded Spill
@@ -9340,10 +9157,10 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_store_dword v14, off, s[16:19], 0 offset:124 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[16:19], 0 offset:128 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
-; NOOPT-NEXT: v_writelane_b32 v33, s0, 9
-; NOOPT-NEXT: v_writelane_b32 v33, s1, 10
+; NOOPT-NEXT: v_writelane_b32 v32, s0, 9
+; NOOPT-NEXT: v_writelane_b32 v32, s1, 10
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
-; NOOPT-NEXT: buffer_store_dword v33, off, s[16:19], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v32, off, s[16:19], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[16:19], 0 offset:8 ; 4-byte Folded Spill
@@ -9365,11 +9182,11 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 11
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 12
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 11
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 12
; NOOPT-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
@@ -9434,100 +9251,92 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_store_dword v14, off, s[16:19], 0 offset:60 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[16:19], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
-; NOOPT-NEXT: v_writelane_b32 v33, s2, 11
-; NOOPT-NEXT: v_writelane_b32 v33, s3, 12
+; NOOPT-NEXT: v_writelane_b32 v32, s2, 11
+; NOOPT-NEXT: v_writelane_b32 v32, s3, 12
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
-; NOOPT-NEXT: buffer_store_dword v33, off, s[16:19], 0 ; 4-byte Folded Spill
+; NOOPT-NEXT: buffer_store_dword v32, off, s[16:19], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_xor_b64 exec, exec, s[0:1]
; NOOPT-NEXT: s_cbranch_execnz .LBB27_1
; NOOPT-NEXT: ; %bb.2:
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 9
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 10
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 9
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 10
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
-; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 5
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 6
-; NOOPT-NEXT: v_readlane_b32 s2, v33, 7
-; NOOPT-NEXT: v_readlane_b32 s3, v33, 8
-; NOOPT-NEXT: v_readlane_b32 s8, v33, 3
-; NOOPT-NEXT: v_readlane_b32 s9, v33, 4
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 5
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 6
+; NOOPT-NEXT: v_readlane_b32 s2, v32, 7
+; NOOPT-NEXT: v_readlane_b32 s3, v32, 8
+; NOOPT-NEXT: v_readlane_b32 s8, v32, 3
+; NOOPT-NEXT: v_readlane_b32 s9, v32, 4
; NOOPT-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v5, off, s[16:19], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[16:19], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[16:19], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[16:19], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[16:19], 0 offset:164 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[16:19], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[16:19], 0 offset:172 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[16:19], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[16:19], 0 offset:180 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[16:19], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[16:19], 0 offset:188 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[16:19], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[16:19], 0 offset:196 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[16:19], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v31, off, s[16:19], 0 offset:204 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 offset:208 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
-; NOOPT-NEXT: v_mov_b32_e32 v6, v20
-; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: v_mov_b32_e32 v8, v18
-; NOOPT-NEXT: v_mov_b32_e32 v0, v17
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
-; NOOPT-NEXT: v_mov_b32_e32 v1, v24
-; NOOPT-NEXT: v_mov_b32_e32 v2, v23
-; NOOPT-NEXT: v_mov_b32_e32 v3, v22
-; NOOPT-NEXT: v_mov_b32_e32 v9, v21
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
-; NOOPT-NEXT: v_mov_b32_e32 v14, v28
-; NOOPT-NEXT: v_mov_b32_e32 v15, v27
-; NOOPT-NEXT: v_mov_b32_e32 v16, v26
-; NOOPT-NEXT: v_mov_b32_e32 v10, v25
+; NOOPT-NEXT: buffer_load_dword v6, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v7, off, s[16:19], 0 offset:152 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v8, off, s[16:19], 0 offset:156 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v9, off, s[16:19], 0 offset:160 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v10, off, s[16:19], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v11, off, s[16:19], 0 offset:168 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v12, off, s[16:19], 0 offset:172 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v13, off, s[16:19], 0 offset:176 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v14, off, s[16:19], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[16:19], 0 offset:184 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v16, off, s[16:19], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:192 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v18, off, s[16:19], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v19, off, s[16:19], 0 offset:200 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v20, off, s[16:19], 0 offset:204 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v21, off, s[16:19], 0 offset:208 ; 4-byte Folded Reload
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v11, v32
-; NOOPT-NEXT: v_mov_b32_e32 v12, v31
-; NOOPT-NEXT: v_mov_b32_e32 v13, v30
-; NOOPT-NEXT: v_mov_b32_e32 v17, v29
-; NOOPT-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18_vgpr19_vgpr20 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v18, v13
-; NOOPT-NEXT: v_mov_b32_e32 v19, v12
-; NOOPT-NEXT: v_mov_b32_e32 v20, v11
+; NOOPT-NEXT: v_mov_b32_e32 v29, v21
+; NOOPT-NEXT: v_mov_b32_e32 v28, v20
+; NOOPT-NEXT: v_mov_b32_e32 v27, v19
+; NOOPT-NEXT: v_mov_b32_e32 v26, v18
+; NOOPT-NEXT: v_mov_b32_e32 v25, v17
+; NOOPT-NEXT: v_mov_b32_e32 v24, v16
+; NOOPT-NEXT: v_mov_b32_e32 v23, v15
+; NOOPT-NEXT: v_mov_b32_e32 v22, v14
+; NOOPT-NEXT: v_mov_b32_e32 v0, v26
+; NOOPT-NEXT: v_mov_b32_e32 v1, v27
+; NOOPT-NEXT: v_mov_b32_e32 v2, v28
+; NOOPT-NEXT: v_mov_b32_e32 v3, v29
; NOOPT-NEXT: s_mov_b64 s[4:5], 32
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
-; NOOPT-NEXT: v_mov_b32_e32 v12, v5
-; NOOPT-NEXT: v_mov_b32_e32 v11, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[17:20], v[11:12], s[4:7], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11_vgpr12_vgpr13 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v11, v16
-; NOOPT-NEXT: v_mov_b32_e32 v12, v15
-; NOOPT-NEXT: v_mov_b32_e32 v13, v14
+; NOOPT-NEXT: v_mov_b32_e32 v31, v5
+; NOOPT-NEXT: v_mov_b32_e32 v30, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[30:31], s[4:7], 0 addr64 offset:16
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: v_mov_b32_e32 v0, v22
+; NOOPT-NEXT: v_mov_b32_e32 v1, v23
+; NOOPT-NEXT: v_mov_b32_e32 v2, v24
+; NOOPT-NEXT: v_mov_b32_e32 v3, v25
+; NOOPT-NEXT: v_mov_b32_e32 v23, v5
+; NOOPT-NEXT: v_mov_b32_e32 v22, v4
+; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[22:23], s[0:3], 0 addr64 offset:32
+; NOOPT-NEXT: ; kill: def $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13 killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13 killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 killed $exec
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: v_mov_b32_e32 v0, v10
+; NOOPT-NEXT: v_mov_b32_e32 v1, v11
+; NOOPT-NEXT: v_mov_b32_e32 v2, v12
+; NOOPT-NEXT: v_mov_b32_e32 v3, v13
; NOOPT-NEXT: v_mov_b32_e32 v15, v5
; NOOPT-NEXT: v_mov_b32_e32 v14, v4
-; NOOPT-NEXT: buffer_store_dwordx4 v[10:13], v[14:15], s[0:3], 0 addr64 offset:32
-; NOOPT-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10_vgpr11_vgpr12 killed $exec
+; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[14:15], s[0:3], 0 addr64 offset:16
; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: v_mov_b32_e32 v10, v3
-; NOOPT-NEXT: v_mov_b32_e32 v11, v2
-; NOOPT-NEXT: v_mov_b32_e32 v12, v1
-; NOOPT-NEXT: v_mov_b32_e32 v1, v4
-; NOOPT-NEXT: v_mov_b32_e32 v2, v5
-; NOOPT-NEXT: buffer_store_dwordx4 v[9:12], v[1:2], s[0:3], 0 addr64 offset:16
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v8
-; NOOPT-NEXT: v_mov_b32_e32 v2, v7
-; NOOPT-NEXT: v_mov_b32_e32 v3, v6
+; NOOPT-NEXT: v_mov_b32_e32 v0, v6
+; NOOPT-NEXT: v_mov_b32_e32 v1, v7
+; NOOPT-NEXT: v_mov_b32_e32 v2, v8
+; NOOPT-NEXT: v_mov_b32_e32 v3, v9
; NOOPT-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
; NOOPT-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
index 9a8fd5bb2119b..e897ae764bf61 100644
--- a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
@@ -8,14 +8,13 @@ define protected amdgpu_kernel void @InferNothing(i32 %a, ptr %b, double %c) {
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_ashr_i32 s7, s6, 31
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
-; CHECK-NEXT: s_lshl_b64 s[2:3], s[6:7], 3
-; CHECK-NEXT: s_add_u32 s0, s0, s2
-; CHECK-NEXT: s_addc_u32 s1, s1, s3
+; CHECK-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
+; CHECK-NEXT: s_add_u32 s0, s0, s4
+; CHECK-NEXT: s_addc_u32 s1, s1, s5
; CHECK-NEXT: s_add_u32 s0, s0, -8
; CHECK-NEXT: s_addc_u32 s1, s1, -1
; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; CHECK-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_wbinvl1_vol
@@ -35,14 +34,13 @@ define protected amdgpu_kernel void @InferFadd(i32 %a, ptr addrspace(1) %b, doub
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_ashr_i32 s7, s6, 31
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
-; CHECK-NEXT: s_lshl_b64 s[2:3], s[6:7], 3
-; CHECK-NEXT: s_add_u32 s0, s0, s2
-; CHECK-NEXT: s_addc_u32 s1, s1, s3
+; CHECK-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
+; CHECK-NEXT: s_add_u32 s0, s0, s4
+; CHECK-NEXT: s_addc_u32 s1, s1, s5
; CHECK-NEXT: s_add_u32 s0, s0, -8
; CHECK-NEXT: s_addc_u32 s1, s1, -1
; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; CHECK-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_wbinvl1_vol
diff --git a/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll b/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
index 5233616c5d134..0d799047ad9b2 100644
--- a/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
@@ -2,8 +2,10 @@
define protected amdgpu_kernel void @test_valu(ptr addrspace(1) noalias noundef writeonly captures(none) %to.coerce, ptr addrspace(1) noalias noundef readonly captures(none) %from.coerce, i32 noundef %k, ptr addrspace(1) noundef writeonly captures(none) %ret.coerce, i32 noundef %length) local_unnamed_addr #0 {
; CHECK-LABEL: test_valu
-; CHECK: s_mul_i32
+; The inline asm counts as the last VALU of the leading group, so the SALU the
+; barrier asks for next lands right after it.
; CHECK: ASMSTART
+; CHECK: s_mul_i32
entry:
%a0 = tail call i32 @llvm.amdgcn.workgroup.id.x()
%mul = shl i32 %a0, 6
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 4f0083bdbd915..5bae91b849808 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -204,11 +204,10 @@ define amdgpu_kernel void @float8_inselt(ptr addrspace(1) %out, <8 x float> %vec
; GCN-LABEL: float8_inselt:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x64
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dword s3, s[4:5], 0x64
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s8
-; GCN-NEXT: s_mov_b32 m0, s2
; GCN-NEXT: s_add_u32 s2, s0, 16
; GCN-NEXT: v_mov_b32_e32 v1, s9
; GCN-NEXT: v_mov_b32_e32 v2, s10
@@ -217,6 +216,7 @@ define amdgpu_kernel void @float8_inselt(ptr addrspace(1) %out, <8 x float> %vec
; GCN-NEXT: v_mov_b32_e32 v5, s13
; GCN-NEXT: v_mov_b32_e32 v6, s14
; GCN-NEXT: v_mov_b32_e32 v7, s15
+; GCN-NEXT: s_mov_b32 m0, s3
; GCN-NEXT: s_addc_u32 s3, s1, 0
; GCN-NEXT: v_movreld_b32_e32 v0, 1.0
; GCN-NEXT: v_mov_b32_e32 v9, s3
@@ -239,26 +239,22 @@ define amdgpu_kernel void @float8_inselt(ptr addrspace(1) %out, <8 x float> %vec
; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
; GCN-O0-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x44
; GCN-O0-NEXT: s_load_dword s2, s[2:3], 0x64
-; GCN-O0-NEXT: v_mov_b32_e32 v0, 1.0
+; GCN-O0-NEXT: v_mov_b32_e32 v8, 1.0
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: v_mov_b32_e32 v14, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v13, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v12, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v11, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v10, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v9, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v8, s5
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v2, s6
+; GCN-O0-NEXT: v_mov_b32_e32 v3, s7
+; GCN-O0-NEXT: v_mov_b32_e32 v4, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v5, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v6, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v7, s11
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v11
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v4
; GCN-O0-NEXT: s_mov_b64 s[6:7], 16
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -268,17 +264,13 @@ define amdgpu_kernel void @float8_inselt(ptr addrspace(1) %out, <8 x float> %vec
; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v7
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
@@ -348,34 +340,38 @@ define amdgpu_kernel void @float16_inselt(ptr addrspace(1) %out, <16 x float> %v
; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
; GCN-O0-NEXT: s_load_dwordx16 s[4:19], s[2:3], 0x64
; GCN-O0-NEXT: s_load_dword s2, s[2:3], 0xa4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, 1.0
+; GCN-O0-NEXT: v_mov_b32_e32 v16, 1.0
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: v_mov_b32_e32 v22, s19
-; GCN-O0-NEXT: v_mov_b32_e32 v21, s18
-; GCN-O0-NEXT: v_mov_b32_e32 v20, s17
-; GCN-O0-NEXT: v_mov_b32_e32 v19, s16
-; GCN-O0-NEXT: v_mov_b32_e32 v18, s15
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s14
-; GCN-O0-NEXT: v_mov_b32_e32 v16, s13
-; GCN-O0-NEXT: v_mov_b32_e32 v15, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v14, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v13, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v12, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v11, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v10, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v9, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v8, s5
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v2, s6
+; GCN-O0-NEXT: v_mov_b32_e32 v3, s7
+; GCN-O0-NEXT: v_mov_b32_e32 v4, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v5, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v6, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v7, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s12
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s13
+; GCN-O0-NEXT: v_mov_b32_e32 v10, s14
+; GCN-O0-NEXT: v_mov_b32_e32 v11, s15
+; GCN-O0-NEXT: v_mov_b32_e32 v12, s16
+; GCN-O0-NEXT: v_mov_b32_e32 v13, s17
+; GCN-O0-NEXT: v_mov_b32_e32 v14, s18
+; GCN-O0-NEXT: v_mov_b32_e32 v15, s19
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v19
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v15
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v13
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v9
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v20
; GCN-O0-NEXT: s_mov_b64 s[6:7], 32
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -394,28 +390,21 @@ define amdgpu_kernel void @float16_inselt(ptr addrspace(1) %out, <16 x float> %v
; GCN-O0-NEXT: s_addc_u32 s8, s5, s8
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v15
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v11
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v24, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[24:25], v[26:29]
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v4
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s6
@@ -424,17 +413,13 @@ define amdgpu_kernel void @float16_inselt(ptr addrspace(1) %out, <16 x float> %v
; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v7
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
@@ -589,7 +574,7 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: s_mov_b32 s34, s38
; GCN-O0-NEXT: s_mov_b32 s35, s37
; GCN-O0-NEXT: ; kill: def $sgpr36 killed $sgpr36 killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s36
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s36
; GCN-O0-NEXT: v_mov_b32_e32 v62, s35
; GCN-O0-NEXT: v_mov_b32_e32 v61, s34
; GCN-O0-NEXT: v_mov_b32_e32 v60, s33
@@ -614,58 +599,78 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: v_mov_b32_e32 v41, s13
; GCN-O0-NEXT: v_mov_b32_e32 v40, s12
; GCN-O0-NEXT: v_mov_b32_e32 v39, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v6, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v5, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v2, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v62
-; GCN-O0-NEXT: v_mov_b32_e32 v9, v61
-; GCN-O0-NEXT: v_mov_b32_e32 v10, v60
-; GCN-O0-NEXT: v_mov_b32_e32 v11, v59
-; GCN-O0-NEXT: v_mov_b32_e32 v12, v58
-; GCN-O0-NEXT: v_mov_b32_e32 v13, v57
-; GCN-O0-NEXT: v_mov_b32_e32 v14, v56
-; GCN-O0-NEXT: v_mov_b32_e32 v15, v55
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v54
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v53
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v52
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v51
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v50
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v49
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v48
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v47
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v46
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v45
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v44
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v43
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v42
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v41
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v40
-; GCN-O0-NEXT: v_mov_b32_e32 v31, v39
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v35, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v36, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v37, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v38, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v38, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v37, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v36, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v35, s7
+; GCN-O0-NEXT: v_mov_b32_e32 v34, s6
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s2
+; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v1, v62
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v61
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v60
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v59
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v58
+; GCN-O0-NEXT: v_mov_b32_e32 v6, v57
+; GCN-O0-NEXT: v_mov_b32_e32 v7, v56
+; GCN-O0-NEXT: v_mov_b32_e32 v8, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v9, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v53
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v52
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v14, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v15, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v24, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v25, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v37
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v36
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v30, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v31, v32
; GCN-O0-NEXT: s_load_dword s2, s[4:5], 0x124
-; GCN-O0-NEXT: v_mov_b32_e32 v0, 1.0
+; GCN-O0-NEXT: v_mov_b32_e32 v32, 1.0
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v38
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v37
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v36
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v35
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
+; GCN-O0-NEXT: v_mov_b32_e32 v47, v31
+; GCN-O0-NEXT: v_mov_b32_e32 v46, v30
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v29
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v28
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v27
+; GCN-O0-NEXT: v_mov_b32_e32 v42, v26
+; GCN-O0-NEXT: v_mov_b32_e32 v41, v25
+; GCN-O0-NEXT: v_mov_b32_e32 v40, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v39, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v33, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v32, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v55, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v54, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v50, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v49, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v48, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v61, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v60, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v59, v53
+; GCN-O0-NEXT: v_mov_b32_e32 v58, v52
; GCN-O0-NEXT: s_mov_b64 s[6:7], 64
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -693,28 +698,21 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: s_addc_u32 s12, s11, s12
; GCN-O0-NEXT: ; kill: def $sgpr10 killed $sgpr10 def $sgpr10_sgpr11
; GCN-O0-NEXT: s_mov_b32 s11, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s11
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v31
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v27
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v57, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v56, s10
+; GCN-O0-NEXT: flat_store_dwordx4 v[56:57], v[58:61]
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v50, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v49, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v48, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[48:49], v[50:53]
+; GCN-O0-NEXT: ; kill: def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v37
+; GCN-O0-NEXT: v_mov_b32_e32 v42, v36
; GCN-O0-NEXT: s_mov_b32 s4, s2
; GCN-O0-NEXT: s_mov_b32 s5, s3
; GCN-O0-NEXT: s_mov_b32 s11, s6
@@ -723,28 +721,29 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: s_addc_u32 s10, s5, s10
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v23
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v19
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v41, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v40, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[40:41], v[42:45]
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v32
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[32:33], v[34:37]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v15
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v13
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v9
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v20
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s8
@@ -761,28 +760,21 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: s_addc_u32 s8, s5, s8
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v15
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v11
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v24, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[24:25], v[26:29]
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v4
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s6
@@ -791,17 +783,13 @@ define amdgpu_kernel void @float32_inselt(ptr addrspace(1) %out, <32 x float> %v
; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v7
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
@@ -2265,46 +2253,18 @@ define amdgpu_kernel void @double5_inselt(ptr addrspace(1) %out, <5 x double> %v
; GCN-O0-NEXT: v_mov_b32_e32 v0, s1
; GCN-O0-NEXT: s_mov_b32 m0, s0
; GCN-O0-NEXT: v_movreld_b32_e32 v1, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v7
-; GCN-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 killed $vgpr17_vgpr18 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v5
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v19
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v4
+; GCN-O0-NEXT: v_mov_b32_e32 v24, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v4
; GCN-O0-NEXT: v_mov_b32_e32 v19, v3
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v19
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v1
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v20
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v27
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v23
-; GCN-O0-NEXT: ; kill: def $vgpr27 killed $vgpr27 def $vgpr27_vgpr28_vgpr29_vgpr30 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v30, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
; GCN-O0-NEXT: s_mov_b64 s[6:7], 16
; GCN-O0-NEXT: s_mov_b32 s0, s2
; GCN-O0-NEXT: s_mov_b32 s1, s3
@@ -2314,17 +2274,13 @@ define amdgpu_kernel void @double5_inselt(ptr addrspace(1) %out, <5 x double> %v
; GCN-O0-NEXT: s_addc_u32 s4, s1, s4
; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GCN-O0-NEXT: s_mov_b32 s1, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v18, s1
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s0
-; GCN-O0-NEXT: flat_store_dwordx4 v[17:18], v[27:30]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v20
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v26, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s0
+; GCN-O0-NEXT: flat_store_dwordx4 v[25:26], v[27:30]
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
; GCN-O0-NEXT: v_mov_b32_e32 v18, s3
; GCN-O0-NEXT: v_mov_b32_e32 v17, s2
; GCN-O0-NEXT: flat_store_dwordx4 v[17:18], v[19:22]
@@ -2416,39 +2372,43 @@ define amdgpu_kernel void @double8_inselt(ptr addrspace(1) %out, <8 x double> %v
; GCN-O0-NEXT: s_mul_i32 s2, s2, s3
; GCN-O0-NEXT: s_mov_b64 s[4:5], 1.0
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v8, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v9, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v10, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v11, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v12, s13
-; GCN-O0-NEXT: v_mov_b32_e32 v13, s14
-; GCN-O0-NEXT: v_mov_b32_e32 v14, s15
-; GCN-O0-NEXT: v_mov_b32_e32 v15, s16
-; GCN-O0-NEXT: v_mov_b32_e32 v16, s17
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s18
-; GCN-O0-NEXT: v_mov_b32_e32 v18, s19
-; GCN-O0-NEXT: v_mov_b32_e32 v19, s20
-; GCN-O0-NEXT: v_mov_b32_e32 v20, s21
-; GCN-O0-NEXT: v_mov_b32_e32 v21, s22
-; GCN-O0-NEXT: v_mov_b32_e32 v22, s23
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v2, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v3, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v4, s12
+; GCN-O0-NEXT: v_mov_b32_e32 v5, s13
+; GCN-O0-NEXT: v_mov_b32_e32 v6, s14
+; GCN-O0-NEXT: v_mov_b32_e32 v7, s15
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s16
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s17
+; GCN-O0-NEXT: v_mov_b32_e32 v10, s18
+; GCN-O0-NEXT: v_mov_b32_e32 v11, s19
+; GCN-O0-NEXT: v_mov_b32_e32 v12, s20
+; GCN-O0-NEXT: v_mov_b32_e32 v13, s21
+; GCN-O0-NEXT: v_mov_b32_e32 v14, s22
+; GCN-O0-NEXT: v_mov_b32_e32 v15, s23
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s3
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v16
; GCN-O0-NEXT: s_mov_b32 s3, 1
; GCN-O0-NEXT: s_add_i32 s2, s2, s3
; GCN-O0-NEXT: s_mov_b32 s3, s5
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s3
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v19
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v15
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v13
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v9
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v20
; GCN-O0-NEXT: s_mov_b64 s[6:7], 32
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -2467,28 +2427,21 @@ define amdgpu_kernel void @double8_inselt(ptr addrspace(1) %out, <8 x double> %v
; GCN-O0-NEXT: s_addc_u32 s8, s5, s8
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v15
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v11
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v24, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[24:25], v[26:29]
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v4
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s6
@@ -2497,17 +2450,13 @@ define amdgpu_kernel void @double8_inselt(ptr addrspace(1) %out, <8 x double> %v
; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v7
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
@@ -2550,7 +2499,7 @@ define amdgpu_kernel void @double7_inselt(ptr addrspace(1) %out, <7 x double> %v
; GCN-NEXT: s_addc_u32 s1, s7, 0
; GCN-NEXT: v_mov_b32_e32 v15, s1
; GCN-NEXT: v_mov_b32_e32 v14, s0
-; GCN-NEXT: s_add_u32 s0, s6, 48
+; GCN-NEXT: s_add_u32 s0, s6, 32
; GCN-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
; GCN-NEXT: s_addc_u32 s1, s7, 0
; GCN-NEXT: v_mov_b32_e32 v4, s6
@@ -2559,90 +2508,89 @@ define amdgpu_kernel void @double7_inselt(ptr addrspace(1) %out, <7 x double> %v
; GCN-NEXT: s_nop 0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: s_add_u32 s0, s6, 32
+; GCN-NEXT: s_add_u32 s0, s6, 48
; GCN-NEXT: s_addc_u32 s1, s7, 0
-; GCN-NEXT: flat_store_dwordx2 v[0:1], v[12:13]
+; GCN-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[12:13]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double7_inselt:
; GCN-O0: ; %bb.0: ; %entry
-; GCN-O0-NEXT: s_mov_b64 s[0:1], s[4:5]
-; GCN-O0-NEXT: s_load_dword s2, s[0:1], 0xa4
-; GCN-O0-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x64
+; GCN-O0-NEXT: s_load_dword s0, s[4:5], 0xa4
+; GCN-O0-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-O0-NEXT: s_mov_b64 s[8:9], 0x64
-; GCN-O0-NEXT: s_mov_b32 s6, s0
-; GCN-O0-NEXT: s_mov_b32 s4, s1
+; GCN-O0-NEXT: s_mov_b32 s6, s4
+; GCN-O0-NEXT: s_mov_b32 s2, s5
; GCN-O0-NEXT: s_mov_b32 s7, s8
-; GCN-O0-NEXT: s_mov_b32 s5, s9
+; GCN-O0-NEXT: s_mov_b32 s3, s9
; GCN-O0-NEXT: s_add_u32 s8, s6, s7
-; GCN-O0-NEXT: s_addc_u32 s4, s4, s5
+; GCN-O0-NEXT: s_addc_u32 s2, s2, s3
; GCN-O0-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9
-; GCN-O0-NEXT: s_mov_b32 s9, s4
+; GCN-O0-NEXT: s_mov_b32 s9, s2
; GCN-O0-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mov_b32 s6, s15
-; GCN-O0-NEXT: s_mov_b32 s4, s14
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s10, s5
-; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: s_mov_b32 s2, s14
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s6
+; GCN-O0-NEXT: s_mov_b32 s10, s3
+; GCN-O0-NEXT: s_mov_b32 s11, s2
; GCN-O0-NEXT: s_mov_b32 s6, s13
-; GCN-O0-NEXT: s_mov_b32 s4, s12
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s12, s5
-; GCN-O0-NEXT: s_mov_b32 s13, s4
-; GCN-O0-NEXT: s_mov_b64 s[4:5], 32
-; GCN-O0-NEXT: s_mov_b32 s6, s8
-; GCN-O0-NEXT: s_mov_b32 s7, s9
-; GCN-O0-NEXT: s_mov_b32 s9, s4
-; GCN-O0-NEXT: s_mov_b32 s8, s5
-; GCN-O0-NEXT: s_add_u32 s6, s6, s9
-; GCN-O0-NEXT: s_addc_u32 s8, s7, s8
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s8
-; GCN-O0-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x10
-; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_mov_b32 s8, s7
+; GCN-O0-NEXT: s_mov_b32 s2, s12
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s6
+; GCN-O0-NEXT: s_mov_b32 s12, s3
+; GCN-O0-NEXT: s_mov_b32 s13, s2
+; GCN-O0-NEXT: s_mov_b64 s[6:7], 32
+; GCN-O0-NEXT: s_mov_b32 s2, s8
+; GCN-O0-NEXT: s_mov_b32 s3, s9
; GCN-O0-NEXT: s_mov_b32 s9, s6
-; GCN-O0-NEXT: s_load_dwordx8 s[24:31], s[0:1], 0x64
+; GCN-O0-NEXT: s_mov_b32 s8, s7
+; GCN-O0-NEXT: s_add_u32 s2, s2, s9
+; GCN-O0-NEXT: s_addc_u32 s8, s3, s8
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s8
+; GCN-O0-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x10
+; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-O0-NEXT: s_mov_b32 s8, s3
+; GCN-O0-NEXT: s_mov_b32 s9, s2
+; GCN-O0-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x64
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mov_b32 s14, s31
-; GCN-O0-NEXT: s_mov_b32 s6, s30
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s14
-; GCN-O0-NEXT: s_mov_b32 s14, s7
-; GCN-O0-NEXT: s_mov_b32 s15, s6
+; GCN-O0-NEXT: s_mov_b32 s2, s30
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s14
+; GCN-O0-NEXT: s_mov_b32 s14, s3
+; GCN-O0-NEXT: s_mov_b32 s15, s2
; GCN-O0-NEXT: s_mov_b32 s16, s29
-; GCN-O0-NEXT: s_mov_b32 s6, s28
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s16
-; GCN-O0-NEXT: s_mov_b32 s16, s7
-; GCN-O0-NEXT: s_mov_b32 s17, s6
+; GCN-O0-NEXT: s_mov_b32 s2, s28
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s16
+; GCN-O0-NEXT: s_mov_b32 s16, s3
+; GCN-O0-NEXT: s_mov_b32 s17, s2
; GCN-O0-NEXT: s_mov_b32 s18, s27
-; GCN-O0-NEXT: s_mov_b32 s6, s26
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s18
-; GCN-O0-NEXT: s_mov_b32 s18, s7
-; GCN-O0-NEXT: s_mov_b32 s19, s6
+; GCN-O0-NEXT: s_mov_b32 s2, s26
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s18
+; GCN-O0-NEXT: s_mov_b32 s18, s3
+; GCN-O0-NEXT: s_mov_b32 s19, s2
; GCN-O0-NEXT: s_mov_b32 s20, s25
-; GCN-O0-NEXT: s_mov_b32 s6, s24
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s20
-; GCN-O0-NEXT: s_mov_b32 s20, s7
-; GCN-O0-NEXT: s_mov_b32 s21, s6
-; GCN-O0-NEXT: ; implicit-def: $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s6, s7
+; GCN-O0-NEXT: s_mov_b32 s2, s24
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s20
+; GCN-O0-NEXT: s_mov_b32 s20, s3
+; GCN-O0-NEXT: s_mov_b32 s21, s2
+; GCN-O0-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s2, s3
; GCN-O0-NEXT: ; implicit-def: $sgpr22_sgpr23
-; GCN-O0-NEXT: s_mov_b32 s7, s22
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s21
+; GCN-O0-NEXT: s_mov_b32 s3, s22
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s21
; GCN-O0-NEXT: v_mov_b32_e32 v30, s20
; GCN-O0-NEXT: v_mov_b32_e32 v29, s19
; GCN-O0-NEXT: v_mov_b32_e32 v28, s18
@@ -2651,123 +2599,95 @@ define amdgpu_kernel void @double7_inselt(ptr addrspace(1) %out, <7 x double> %v
; GCN-O0-NEXT: v_mov_b32_e32 v25, s15
; GCN-O0-NEXT: v_mov_b32_e32 v24, s14
; GCN-O0-NEXT: v_mov_b32_e32 v23, s13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v5, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v2, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s6
-; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v9, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v10, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v11, v27
-; GCN-O0-NEXT: v_mov_b32_e32 v12, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v13, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v14, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v15, v23
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v0
-; GCN-O0-NEXT: s_load_dword s0, s[0:1], 0xa4
-; GCN-O0-NEXT: s_mov_b32 s1, 2
+; GCN-O0-NEXT: v_mov_b32_e32 v22, s12
+; GCN-O0-NEXT: v_mov_b32_e32 v21, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v20, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v19, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v18, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
+; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v30
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v29
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v28
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v27
+; GCN-O0-NEXT: v_mov_b32_e32 v6, v26
+; GCN-O0-NEXT: v_mov_b32_e32 v7, v25
+; GCN-O0-NEXT: v_mov_b32_e32 v8, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v9, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v14, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v15, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v0
+; GCN-O0-NEXT: s_load_dword s2, s[4:5], 0xa4
+; GCN-O0-NEXT: s_mov_b32 s3, 2
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_mul_i32 s0, s0, s1
-; GCN-O0-NEXT: s_mov_b64 s[6:7], 1.0
-; GCN-O0-NEXT: s_mov_b32 s1, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s1
-; GCN-O0-NEXT: s_mov_b32 m0, s0
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: s_mov_b32 s1, 1
-; GCN-O0-NEXT: s_add_i32 s0, s0, s1
+; GCN-O0-NEXT: s_mul_i32 s2, s2, s3
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 1.0
+; GCN-O0-NEXT: s_mov_b32 s3, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: s_mov_b32 m0, s2
+; GCN-O0-NEXT: v_movreld_b32_e32 v1, v0
+; GCN-O0-NEXT: s_mov_b32 s3, 1
+; GCN-O0-NEXT: s_add_i32 s2, s2, s3
+; GCN-O0-NEXT: s_mov_b32 s3, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: s_mov_b32 m0, s2
+; GCN-O0-NEXT: v_movreld_b32_e32 v1, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v24, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v4
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v30, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 16
+; GCN-O0-NEXT: s_mov_b32 s2, s0
+; GCN-O0-NEXT: s_mov_b32 s3, s1
+; GCN-O0-NEXT: s_mov_b32 s9, s4
+; GCN-O0-NEXT: s_mov_b32 s8, s5
+; GCN-O0-NEXT: s_add_u32 s2, s2, s9
+; GCN-O0-NEXT: s_addc_u32 s8, s3, s8
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v26, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[25:26], v[27:30]
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v18, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s0
+; GCN-O0-NEXT: flat_store_dwordx4 v[17:18], v[19:22]
+; GCN-O0-NEXT: s_nop 0
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v9
+; GCN-O0-NEXT: s_mov_b32 s2, s0
+; GCN-O0-NEXT: s_mov_b32 s0, s1
+; GCN-O0-NEXT: s_mov_b32 s3, s6
; GCN-O0-NEXT: s_mov_b32 s1, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s1
-; GCN-O0-NEXT: s_mov_b32 m0, s0
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v17
-; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v2
-; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v15
+; GCN-O0-NEXT: s_add_u32 s2, s2, s3
+; GCN-O0-NEXT: s_addc_u32 s0, s0, s1
+; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GCN-O0-NEXT: s_mov_b32 s3, s0
+; GCN-O0-NEXT: v_mov_b32_e32 v18, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[17:18], v[19:22]
+; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v13
; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v3
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v13
-; GCN-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v5
-; GCN-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v11
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v31, v23
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v9
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v23
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v7
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v24
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 killed $vgpr23_vgpr24 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v31
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v31, v27
-; GCN-O0-NEXT: ; kill: def $vgpr31 killed $vgpr31 def $vgpr31_vgpr32_vgpr33_vgpr34 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v3
-; GCN-O0-NEXT: s_mov_b64 s[6:7], 16
-; GCN-O0-NEXT: s_mov_b32 s0, s2
-; GCN-O0-NEXT: s_mov_b32 s1, s3
-; GCN-O0-NEXT: s_mov_b32 s9, s6
-; GCN-O0-NEXT: s_mov_b32 s8, s7
-; GCN-O0-NEXT: s_add_u32 s0, s0, s9
-; GCN-O0-NEXT: s_addc_u32 s8, s1, s8
-; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GCN-O0-NEXT: s_mov_b32 s1, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s1
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s0
-; GCN-O0-NEXT: flat_store_dwordx4 v[3:4], v[31:34]
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v24
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 killed $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24_vgpr25_vgpr26 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
-; GCN-O0-NEXT: flat_store_dwordx4 v[3:4], v[23:26]
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v7, v19
-; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v0
; GCN-O0-NEXT: s_mov_b32 s0, s2
; GCN-O0-NEXT: s_mov_b32 s1, s3
; GCN-O0-NEXT: s_mov_b32 s3, s4
@@ -2776,24 +2696,9 @@ define amdgpu_kernel void @double7_inselt(ptr addrspace(1) %out, <7 x double> %v
; GCN-O0-NEXT: s_addc_u32 s2, s1, s2
; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GCN-O0-NEXT: s_mov_b32 s1, s2
-; GCN-O0-NEXT: s_mov_b32 s2, s0
-; GCN-O0-NEXT: s_mov_b32 s3, s1
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s4, s7
-; GCN-O0-NEXT: s_add_u32 s2, s2, s5
-; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
-; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
-; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
-; GCN-O0-NEXT: flat_store_dwordx2 v[3:4], v[7:8]
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GCN-O0-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-O0-NEXT: s_endpgm
entry:
%v = insertelement <7 x double> %vec, double 1.000000e+00, i32 %sel
@@ -2948,7 +2853,7 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: s_mov_b32 s34, s38
; GCN-O0-NEXT: s_mov_b32 s35, s37
; GCN-O0-NEXT: ; kill: def $sgpr36 killed $sgpr36 killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; GCN-O0-NEXT: v_mov_b32_e32 v7, s36
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s36
; GCN-O0-NEXT: v_mov_b32_e32 v62, s35
; GCN-O0-NEXT: v_mov_b32_e32 v61, s34
; GCN-O0-NEXT: v_mov_b32_e32 v60, s33
@@ -2973,68 +2878,88 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v41, s13
; GCN-O0-NEXT: v_mov_b32_e32 v40, s12
; GCN-O0-NEXT: v_mov_b32_e32 v39, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v6, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v5, s9
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v2, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v62
-; GCN-O0-NEXT: v_mov_b32_e32 v9, v61
-; GCN-O0-NEXT: v_mov_b32_e32 v10, v60
-; GCN-O0-NEXT: v_mov_b32_e32 v11, v59
-; GCN-O0-NEXT: v_mov_b32_e32 v12, v58
-; GCN-O0-NEXT: v_mov_b32_e32 v13, v57
-; GCN-O0-NEXT: v_mov_b32_e32 v14, v56
-; GCN-O0-NEXT: v_mov_b32_e32 v15, v55
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v54
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v53
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v52
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v51
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v50
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v49
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v48
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v47
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v46
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v45
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v44
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v43
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v42
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v41
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v40
-; GCN-O0-NEXT: v_mov_b32_e32 v31, v39
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v35, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v36, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v37, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v38, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v38, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v37, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v36, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v35, s7
+; GCN-O0-NEXT: v_mov_b32_e32 v34, s6
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s2
+; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v1, v62
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v61
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v60
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v59
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v58
+; GCN-O0-NEXT: v_mov_b32_e32 v6, v57
+; GCN-O0-NEXT: v_mov_b32_e32 v7, v56
+; GCN-O0-NEXT: v_mov_b32_e32 v8, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v9, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v53
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v52
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v14, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v15, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v24, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v25, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v37
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v36
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v30, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v31, v32
; GCN-O0-NEXT: s_load_dword s2, s[4:5], 0x124
; GCN-O0-NEXT: s_mov_b32 s3, 2
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mul_i32 s2, s2, s3
; GCN-O0-NEXT: s_mov_b64 s[4:5], 1.0
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s3
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
; GCN-O0-NEXT: s_mov_b32 s3, 1
; GCN-O0-NEXT: s_add_i32 s2, s2, s3
; GCN-O0-NEXT: s_mov_b32 s3, s5
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s3
; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v7, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v38
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v37
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v36
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v35
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
+; GCN-O0-NEXT: v_mov_b32_e32 v47, v31
+; GCN-O0-NEXT: v_mov_b32_e32 v46, v30
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v29
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v28
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v27
+; GCN-O0-NEXT: v_mov_b32_e32 v42, v26
+; GCN-O0-NEXT: v_mov_b32_e32 v41, v25
+; GCN-O0-NEXT: v_mov_b32_e32 v40, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v39, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v33, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v32, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v55, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v54, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v50, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v49, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v48, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v61, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v60, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v59, v53
+; GCN-O0-NEXT: v_mov_b32_e32 v58, v52
; GCN-O0-NEXT: s_mov_b64 s[6:7], 64
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -3062,28 +2987,21 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: s_addc_u32 s12, s11, s12
; GCN-O0-NEXT: ; kill: def $sgpr10 killed $sgpr10 def $sgpr10_sgpr11
; GCN-O0-NEXT: s_mov_b32 s11, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s11
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v31
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v27
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v57, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v56, s10
+; GCN-O0-NEXT: flat_store_dwordx4 v[56:57], v[58:61]
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v50, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v49, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v48, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[48:49], v[50:53]
+; GCN-O0-NEXT: ; kill: def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v37
+; GCN-O0-NEXT: v_mov_b32_e32 v42, v36
; GCN-O0-NEXT: s_mov_b32 s4, s2
; GCN-O0-NEXT: s_mov_b32 s5, s3
; GCN-O0-NEXT: s_mov_b32 s11, s6
@@ -3092,28 +3010,29 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: s_addc_u32 s10, s5, s10
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v23
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v19
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v41, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v40, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[40:41], v[42:45]
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v32
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v32, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[32:33], v[34:37]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v15
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v13
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v9
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v20
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s8
@@ -3130,28 +3049,21 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: s_addc_u32 s8, s5, s8
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v15
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v12
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v11
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v25, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v24, s4
+; GCN-O0-NEXT: flat_store_dwordx4 v[24:25], v[26:29]
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
+; GCN-O0-NEXT: ; kill: def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v4
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s6
@@ -3160,17 +3072,13 @@ define amdgpu_kernel void @double16_inselt(ptr addrspace(1) %out, <16 x double>
; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
-; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GCN-O0-NEXT: v_mov_b32_e32 v0, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v8
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v7
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v9, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v8, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
@@ -3255,52 +3163,45 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-NEXT: s_nop 0
; GCN-NEXT: v_mov_b32_e32 v9, s3
; GCN-NEXT: v_mov_b32_e32 v8, s2
-; GCN-NEXT: s_add_u32 s2, s0, 0x70
+; GCN-NEXT: s_add_u32 s2, s0, 0x60
; GCN-NEXT: s_addc_u32 s3, s1, 0
; GCN-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCN-NEXT: s_nop 0
; GCN-NEXT: v_mov_b32_e32 v5, s1
; GCN-NEXT: v_mov_b32_e32 v4, s0
-; GCN-NEXT: s_add_u32 s0, s0, 0x60
+; GCN-NEXT: s_add_u32 s0, s0, 0x70
; GCN-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NEXT: s_addc_u32 s1, s1, 0
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NEXT: flat_store_dwordx2 v[0:1], v[28:29]
+; GCN-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[28:29]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double15_inselt:
; GCN-O0: ; %bb.0: ; %entry
-; GCN-O0-NEXT: s_mov_b32 s56, SCRATCH_RSRC_DWORD0
-; GCN-O0-NEXT: s_mov_b32 s57, SCRATCH_RSRC_DWORD1
-; GCN-O0-NEXT: s_mov_b32 s58, -1
-; GCN-O0-NEXT: s_mov_b32 s59, 0xe80000
-; GCN-O0-NEXT: s_add_u32 s56, s56, s11
-; GCN-O0-NEXT: s_addc_u32 s57, s57, 0
-; GCN-O0-NEXT: s_mov_b64 s[2:3], s[4:5]
-; GCN-O0-NEXT: s_load_dwordx16 s[4:19], s[2:3], 0xa4
-; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-O0-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0xa4
; GCN-O0-NEXT: s_mov_b64 s[6:7], 0xa4
-; GCN-O0-NEXT: s_mov_b32 s4, s2
-; GCN-O0-NEXT: s_mov_b32 s0, s3
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s1, s7
-; GCN-O0-NEXT: s_add_u32 s4, s4, s5
-; GCN-O0-NEXT: s_addc_u32 s0, s0, s1
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s0
-; GCN-O0-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; GCN-O0-NEXT: s_load_dword s0, s[2:3], 0x124
+; GCN-O0-NEXT: s_mov_b32 s0, s4
+; GCN-O0-NEXT: s_mov_b32 s1, s5
+; GCN-O0-NEXT: s_mov_b32 s3, s6
+; GCN-O0-NEXT: s_mov_b32 s2, s7
+; GCN-O0-NEXT: s_add_u32 s0, s0, s3
+; GCN-O0-NEXT: s_addc_u32 s2, s1, s2
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s2
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
+; GCN-O0-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x40
+; GCN-O0-NEXT: s_load_dword s2, s[4:5], 0x124
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
+; GCN-O0-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-O0-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GCN-O0-NEXT: s_mov_b64 s[8:9], 64
-; GCN-O0-NEXT: s_mov_b32 s10, s4
-; GCN-O0-NEXT: s_mov_b32 s6, s5
+; GCN-O0-NEXT: s_mov_b32 s10, s0
+; GCN-O0-NEXT: s_mov_b32 s6, s1
; GCN-O0-NEXT: s_mov_b32 s11, s8
; GCN-O0-NEXT: s_mov_b32 s7, s9
; GCN-O0-NEXT: s_add_u32 s10, s10, s11
@@ -3321,100 +3222,100 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: s_mov_b32 s7, s14
; GCN-O0-NEXT: s_mov_b32 s14, s7
; GCN-O0-NEXT: s_mov_b32 s15, s6
-; GCN-O0-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x40
+; GCN-O0-NEXT: s_load_dwordx8 s[24:31], s[0:1], 0x40
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mov_b32 s6, s31
-; GCN-O0-NEXT: s_mov_b32 s4, s30
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s16, s5
-; GCN-O0-NEXT: s_mov_b32 s17, s4
+; GCN-O0-NEXT: s_mov_b32 s0, s30
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s6
+; GCN-O0-NEXT: s_mov_b32 s16, s1
+; GCN-O0-NEXT: s_mov_b32 s17, s0
; GCN-O0-NEXT: s_mov_b32 s6, s29
-; GCN-O0-NEXT: s_mov_b32 s4, s28
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s18, s5
-; GCN-O0-NEXT: s_mov_b32 s19, s4
+; GCN-O0-NEXT: s_mov_b32 s0, s28
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s6
+; GCN-O0-NEXT: s_mov_b32 s18, s1
+; GCN-O0-NEXT: s_mov_b32 s19, s0
; GCN-O0-NEXT: s_mov_b32 s6, s27
-; GCN-O0-NEXT: s_mov_b32 s4, s26
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s20, s5
-; GCN-O0-NEXT: s_mov_b32 s21, s4
+; GCN-O0-NEXT: s_mov_b32 s0, s26
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s6
+; GCN-O0-NEXT: s_mov_b32 s20, s1
+; GCN-O0-NEXT: s_mov_b32 s21, s0
; GCN-O0-NEXT: s_mov_b32 s6, s25
-; GCN-O0-NEXT: s_mov_b32 s4, s24
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s22, s5
-; GCN-O0-NEXT: s_mov_b32 s23, s4
-; GCN-O0-NEXT: s_mov_b64 s[4:5], 32
-; GCN-O0-NEXT: s_mov_b32 s6, s10
-; GCN-O0-NEXT: s_mov_b32 s7, s11
-; GCN-O0-NEXT: s_mov_b32 s11, s4
-; GCN-O0-NEXT: s_mov_b32 s10, s5
-; GCN-O0-NEXT: s_add_u32 s6, s6, s11
-; GCN-O0-NEXT: s_addc_u32 s10, s7, s10
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s10
-; GCN-O0-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x10
-; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_mov_b32 s10, s7
+; GCN-O0-NEXT: s_mov_b32 s0, s24
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s6
+; GCN-O0-NEXT: s_mov_b32 s22, s1
+; GCN-O0-NEXT: s_mov_b32 s23, s0
+; GCN-O0-NEXT: s_mov_b64 s[6:7], 32
+; GCN-O0-NEXT: s_mov_b32 s0, s10
+; GCN-O0-NEXT: s_mov_b32 s1, s11
; GCN-O0-NEXT: s_mov_b32 s11, s6
-; GCN-O0-NEXT: s_load_dwordx16 s[40:55], s[2:3], 0xa4
+; GCN-O0-NEXT: s_mov_b32 s10, s7
+; GCN-O0-NEXT: s_add_u32 s0, s0, s11
+; GCN-O0-NEXT: s_addc_u32 s10, s1, s10
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s10
+; GCN-O0-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x10
+; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-O0-NEXT: s_mov_b32 s10, s1
+; GCN-O0-NEXT: s_mov_b32 s11, s0
+; GCN-O0-NEXT: s_load_dwordx16 s[40:55], s[4:5], 0xa4
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
; GCN-O0-NEXT: s_mov_b32 s24, s55
-; GCN-O0-NEXT: s_mov_b32 s6, s54
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s24
-; GCN-O0-NEXT: s_mov_b32 s24, s7
-; GCN-O0-NEXT: s_mov_b32 s25, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s54
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s24
+; GCN-O0-NEXT: s_mov_b32 s24, s1
+; GCN-O0-NEXT: s_mov_b32 s25, s0
; GCN-O0-NEXT: s_mov_b32 s26, s53
-; GCN-O0-NEXT: s_mov_b32 s6, s52
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s26
-; GCN-O0-NEXT: s_mov_b32 s26, s7
-; GCN-O0-NEXT: s_mov_b32 s27, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s52
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s26
+; GCN-O0-NEXT: s_mov_b32 s26, s1
+; GCN-O0-NEXT: s_mov_b32 s27, s0
; GCN-O0-NEXT: s_mov_b32 s28, s51
-; GCN-O0-NEXT: s_mov_b32 s6, s50
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s28
-; GCN-O0-NEXT: s_mov_b32 s28, s7
-; GCN-O0-NEXT: s_mov_b32 s29, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s50
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s28
+; GCN-O0-NEXT: s_mov_b32 s28, s1
+; GCN-O0-NEXT: s_mov_b32 s29, s0
; GCN-O0-NEXT: s_mov_b32 s30, s49
-; GCN-O0-NEXT: s_mov_b32 s6, s48
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s30
-; GCN-O0-NEXT: s_mov_b32 s30, s7
-; GCN-O0-NEXT: s_mov_b32 s31, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s48
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s30
+; GCN-O0-NEXT: s_mov_b32 s30, s1
+; GCN-O0-NEXT: s_mov_b32 s31, s0
; GCN-O0-NEXT: s_mov_b32 s33, s47
-; GCN-O0-NEXT: s_mov_b32 s6, s46
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s33
-; GCN-O0-NEXT: s_mov_b32 s33, s7
-; GCN-O0-NEXT: s_mov_b32 s34, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s46
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s33
+; GCN-O0-NEXT: s_mov_b32 s33, s1
+; GCN-O0-NEXT: s_mov_b32 s34, s0
; GCN-O0-NEXT: s_mov_b32 s35, s45
-; GCN-O0-NEXT: s_mov_b32 s6, s44
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s35
-; GCN-O0-NEXT: s_mov_b32 s35, s7
-; GCN-O0-NEXT: s_mov_b32 s36, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s44
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s35
+; GCN-O0-NEXT: s_mov_b32 s35, s1
+; GCN-O0-NEXT: s_mov_b32 s36, s0
; GCN-O0-NEXT: s_mov_b32 s37, s43
-; GCN-O0-NEXT: s_mov_b32 s6, s42
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s37
-; GCN-O0-NEXT: s_mov_b32 s37, s7
-; GCN-O0-NEXT: s_mov_b32 s38, s6
+; GCN-O0-NEXT: s_mov_b32 s0, s42
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s37
+; GCN-O0-NEXT: s_mov_b32 s37, s1
+; GCN-O0-NEXT: s_mov_b32 s38, s0
; GCN-O0-NEXT: s_mov_b32 s39, s41
-; GCN-O0-NEXT: s_mov_b32 s6, s40
-; GCN-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s7, s39
-; GCN-O0-NEXT: s_mov_b32 s39, s7
-; GCN-O0-NEXT: s_mov_b32 s40, s6
-; GCN-O0-NEXT: ; implicit-def: $sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s6, s7
+; GCN-O0-NEXT: s_mov_b32 s0, s40
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s39
+; GCN-O0-NEXT: s_mov_b32 s39, s1
+; GCN-O0-NEXT: s_mov_b32 s40, s0
+; GCN-O0-NEXT: ; implicit-def: $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s0, s1
; GCN-O0-NEXT: ; implicit-def: $sgpr42_sgpr43
-; GCN-O0-NEXT: s_mov_b32 s7, s42
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s40
+; GCN-O0-NEXT: s_mov_b32 s1, s42
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s40
; GCN-O0-NEXT: v_mov_b32_e32 v62, s39
; GCN-O0-NEXT: v_mov_b32_e32 v61, s38
; GCN-O0-NEXT: v_mov_b32_e32 v60, s37
@@ -3444,385 +3345,191 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v36, s12
; GCN-O0-NEXT: v_mov_b32_e32 v35, s11
; GCN-O0-NEXT: v_mov_b32_e32 v34, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v33, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v32, s6
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v62
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v61
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v60
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v59
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v58
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v57
-; GCN-O0-NEXT: v_mov_b32_e32 v7, v56
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v55
-; GCN-O0-NEXT: v_mov_b32_e32 v9, v54
-; GCN-O0-NEXT: v_mov_b32_e32 v10, v53
-; GCN-O0-NEXT: v_mov_b32_e32 v11, v52
-; GCN-O0-NEXT: v_mov_b32_e32 v12, v51
-; GCN-O0-NEXT: v_mov_b32_e32 v13, v50
-; GCN-O0-NEXT: v_mov_b32_e32 v14, v49
-; GCN-O0-NEXT: v_mov_b32_e32 v15, v48
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v47
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v46
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v45
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v44
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v43
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v42
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v41
-; GCN-O0-NEXT: v_mov_b32_e32 v23, v40
-; GCN-O0-NEXT: v_mov_b32_e32 v24, v39
-; GCN-O0-NEXT: v_mov_b32_e32 v25, v38
-; GCN-O0-NEXT: v_mov_b32_e32 v26, v37
-; GCN-O0-NEXT: v_mov_b32_e32 v27, v36
-; GCN-O0-NEXT: v_mov_b32_e32 v28, v35
-; GCN-O0-NEXT: v_mov_b32_e32 v29, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v31, v32
-; GCN-O0-NEXT: s_load_dword s2, s[2:3], 0x124
-; GCN-O0-NEXT: s_mov_b32 s3, 2
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
+; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v62
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v61
+; GCN-O0-NEXT: v_mov_b32_e32 v4, v60
+; GCN-O0-NEXT: v_mov_b32_e32 v5, v59
+; GCN-O0-NEXT: v_mov_b32_e32 v6, v58
+; GCN-O0-NEXT: v_mov_b32_e32 v7, v57
+; GCN-O0-NEXT: v_mov_b32_e32 v8, v56
+; GCN-O0-NEXT: v_mov_b32_e32 v9, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v10, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v11, v53
+; GCN-O0-NEXT: v_mov_b32_e32 v12, v52
+; GCN-O0-NEXT: v_mov_b32_e32 v13, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v14, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v15, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v16, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v17, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v18, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v19, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v20, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v21, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v22, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v23, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v24, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v25, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v26, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v27, v37
+; GCN-O0-NEXT: v_mov_b32_e32 v28, v36
+; GCN-O0-NEXT: v_mov_b32_e32 v29, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v30, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v31, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v32, v0
+; GCN-O0-NEXT: s_load_dword s0, s[4:5], 0x124
+; GCN-O0-NEXT: s_mov_b32 s1, 2
; GCN-O0-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-O0-NEXT: s_mul_i32 s2, s2, s3
-; GCN-O0-NEXT: s_mov_b64 s[6:7], 1.0
-; GCN-O0-NEXT: s_mov_b32 s3, s6
-; GCN-O0-NEXT: v_mov_b32_e32 v32, s3
-; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
-; GCN-O0-NEXT: s_mov_b32 s3, 1
-; GCN-O0-NEXT: s_add_i32 s2, s2, s3
-; GCN-O0-NEXT: s_mov_b32 s3, s7
-; GCN-O0-NEXT: v_mov_b32_e32 v32, s3
-; GCN-O0-NEXT: s_mov_b32 m0, s2
-; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
-; GCN-O0-NEXT: buffer_store_dword v0, off, s[56:59], 0 offset:28 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v1, off, s[56:59], 0 offset:32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v2, off, s[56:59], 0 offset:36 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v3, off, s[56:59], 0 offset:40 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v4, off, s[56:59], 0 offset:44 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v5, off, s[56:59], 0 offset:48 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v6, off, s[56:59], 0 offset:52 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[56:59], 0 offset:56 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v8, off, s[56:59], 0 offset:60 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v9, off, s[56:59], 0 offset:64 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v10, off, s[56:59], 0 offset:68 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v11, off, s[56:59], 0 offset:72 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v12, off, s[56:59], 0 offset:76 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v13, off, s[56:59], 0 offset:80 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v14, off, s[56:59], 0 offset:84 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v15, off, s[56:59], 0 offset:88 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v16, off, s[56:59], 0 offset:92 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v17, off, s[56:59], 0 offset:96 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v18, off, s[56:59], 0 offset:100 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v19, off, s[56:59], 0 offset:104 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v20, off, s[56:59], 0 offset:108 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v21, off, s[56:59], 0 offset:112 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v22, off, s[56:59], 0 offset:116 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v23, off, s[56:59], 0 offset:120 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v24, off, s[56:59], 0 offset:124 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v25, off, s[56:59], 0 offset:128 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v26, off, s[56:59], 0 offset:132 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v27, off, s[56:59], 0 offset:136 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v28, off, s[56:59], 0 offset:140 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v29, off, s[56:59], 0 offset:144 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v30, off, s[56:59], 0 offset:148 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_store_dword v31, off, s[56:59], 0 offset:152 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v27
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v26
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:4 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v24
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:24 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:8 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v23
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v22
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:12 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:16 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v20
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:20 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v47, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v19
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v18
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v48, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v49, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v16
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v50, v33
-; GCN-O0-NEXT: v_mov_b32_e32 v39, v32
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v15
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v14
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:160 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:164 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v12
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:168 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:172 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v11
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v10
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:176 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:180 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v8
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:184 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:188 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v7
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v6
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:192 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:196 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v4
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:200 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:204 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v2
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 def $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v33, v34
-; GCN-O0-NEXT: v_mov_b32_e32 v34, v33
-; GCN-O0-NEXT: buffer_store_dword v34, off, s[56:59], 0 offset:208 ; 4-byte Folded Spill
-; GCN-O0-NEXT: ; kill: def $vgpr32 killed $vgpr32 killed $vgpr32_vgpr33 killed $exec
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:212 ; 4-byte Folded Spill
-; GCN-O0-NEXT: v_mov_b32_e32 v32, v1
-; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:156 ; 4-byte Folded Spill
-; GCN-O0-NEXT: buffer_load_dword v29, off, s[56:59], 0 offset:212 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v28, off, s[56:59], 0 offset:208 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v27, off, s[56:59], 0 offset:204 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v26, off, s[56:59], 0 offset:200 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v25, off, s[56:59], 0 offset:196 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v24, off, s[56:59], 0 offset:192 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v23, off, s[56:59], 0 offset:188 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v22, off, s[56:59], 0 offset:184 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v21, off, s[56:59], 0 offset:180 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v20, off, s[56:59], 0 offset:176 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v19, off, s[56:59], 0 offset:172 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v18, off, s[56:59], 0 offset:168 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v17, off, s[56:59], 0 offset:164 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v16, off, s[56:59], 0 offset:160 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:156 ; 4-byte Folded Reload
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v30, v1
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v30
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v29
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v28
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v27
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v26
-; GCN-O0-NEXT: v_mov_b32_e32 v6, v25
-; GCN-O0-NEXT: v_mov_b32_e32 v7, v24
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v23
-; GCN-O0-NEXT: v_mov_b32_e32 v9, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v10, v21
-; GCN-O0-NEXT: v_mov_b32_e32 v11, v20
-; GCN-O0-NEXT: v_mov_b32_e32 v12, v19
-; GCN-O0-NEXT: v_mov_b32_e32 v13, v18
-; GCN-O0-NEXT: v_mov_b32_e32 v14, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v15, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v15
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v14
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v13
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v12
-; GCN-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19_vgpr20_vgpr21 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v16
-; GCN-O0-NEXT: s_mov_b32 s2, s0
-; GCN-O0-NEXT: s_mov_b32 s3, s1
-; GCN-O0-NEXT: s_mov_b32 s7, s4
-; GCN-O0-NEXT: s_mov_b32 s6, s5
-; GCN-O0-NEXT: s_add_u32 s2, s2, s7
-; GCN-O0-NEXT: s_addc_u32 s6, s3, s6
-; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
-; GCN-O0-NEXT: s_mov_b32 s3, s6
-; GCN-O0-NEXT: s_mov_b64 s[6:7], 16
-; GCN-O0-NEXT: s_mov_b32 s10, s2
-; GCN-O0-NEXT: s_mov_b32 s11, s3
-; GCN-O0-NEXT: s_mov_b32 s13, s6
-; GCN-O0-NEXT: s_mov_b32 s12, s7
+; GCN-O0-NEXT: s_mul_i32 s0, s0, s1
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 1.0
+; GCN-O0-NEXT: s_mov_b32 s1, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s1
+; GCN-O0-NEXT: s_mov_b32 m0, s0
+; GCN-O0-NEXT: v_movreld_b32_e32 v1, v0
+; GCN-O0-NEXT: s_mov_b32 s1, 1
+; GCN-O0-NEXT: s_add_i32 s0, s0, s1
+; GCN-O0-NEXT: s_mov_b32 s1, s5
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s1
+; GCN-O0-NEXT: s_mov_b32 m0, s0
+; GCN-O0-NEXT: v_movreld_b32_e32 v1, v0
+; GCN-O0-NEXT: v_mov_b32_e32 v48, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v47, v15
+; GCN-O0-NEXT: v_mov_b32_e32 v46, v14
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v13
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v12
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v11
+; GCN-O0-NEXT: v_mov_b32_e32 v42, v10
+; GCN-O0-NEXT: v_mov_b32_e32 v41, v9
+; GCN-O0-NEXT: v_mov_b32_e32 v40, v8
+; GCN-O0-NEXT: v_mov_b32_e32 v39, v7
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v6
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v5
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v4
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v2
+; GCN-O0-NEXT: v_mov_b32_e32 v33, v1
+; GCN-O0-NEXT: v_mov_b32_e32 v56, v48
+; GCN-O0-NEXT: v_mov_b32_e32 v55, v47
+; GCN-O0-NEXT: v_mov_b32_e32 v54, v46
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v45
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v44
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v43
+; GCN-O0-NEXT: v_mov_b32_e32 v50, v42
+; GCN-O0-NEXT: v_mov_b32_e32 v49, v41
+; GCN-O0-NEXT: v_mov_b32_e32 v62, v56
+; GCN-O0-NEXT: v_mov_b32_e32 v61, v55
+; GCN-O0-NEXT: v_mov_b32_e32 v60, v54
+; GCN-O0-NEXT: v_mov_b32_e32 v59, v53
+; GCN-O0-NEXT: s_mov_b32 s0, s2
+; GCN-O0-NEXT: s_mov_b32 s1, s3
+; GCN-O0-NEXT: s_mov_b32 s5, s6
+; GCN-O0-NEXT: s_mov_b32 s4, s7
+; GCN-O0-NEXT: s_add_u32 s0, s0, s5
+; GCN-O0-NEXT: s_addc_u32 s4, s1, s4
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s4
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 16
+; GCN-O0-NEXT: s_mov_b32 s10, s0
+; GCN-O0-NEXT: s_mov_b32 s11, s1
+; GCN-O0-NEXT: s_mov_b32 s13, s4
+; GCN-O0-NEXT: s_mov_b32 s12, s5
; GCN-O0-NEXT: s_add_u32 s10, s10, s13
; GCN-O0-NEXT: s_addc_u32 s12, s11, s12
; GCN-O0-NEXT: ; kill: def $sgpr10 killed $sgpr10 def $sgpr10_sgpr11
; GCN-O0-NEXT: s_mov_b32 s11, s12
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s11
-; GCN-O0-NEXT: v_mov_b32_e32 v16, s10
-; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v11
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v10
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v9
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v8
-; GCN-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19_vgpr20_vgpr21 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v16
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
-; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
-; GCN-O0-NEXT: v_mov_b32_e32 v16, v7
-; GCN-O0-NEXT: v_mov_b32_e32 v17, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v22, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v18, v4
-; GCN-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19_vgpr20_vgpr21 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v19, v22
-; GCN-O0-NEXT: v_mov_b32_e32 v20, v17
-; GCN-O0-NEXT: v_mov_b32_e32 v21, v16
+; GCN-O0-NEXT: v_mov_b32_e32 v58, s11
+; GCN-O0-NEXT: v_mov_b32_e32 v57, s10
+; GCN-O0-NEXT: flat_store_dwordx4 v[57:58], v[59:62]
+; GCN-O0-NEXT: v_mov_b32_e32 v54, v52
+; GCN-O0-NEXT: v_mov_b32_e32 v53, v51
+; GCN-O0-NEXT: v_mov_b32_e32 v52, v50
+; GCN-O0-NEXT: v_mov_b32_e32 v51, v49
+; GCN-O0-NEXT: v_mov_b32_e32 v50, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v49, s0
+; GCN-O0-NEXT: flat_store_dwordx4 v[49:50], v[51:54]
+; GCN-O0-NEXT: ; kill: def $vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40 killed $vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40 killed $vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v46, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v37
+; GCN-O0-NEXT: s_mov_b32 s0, s2
+; GCN-O0-NEXT: s_mov_b32 s1, s3
+; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: s_mov_b32 s10, s5
+; GCN-O0-NEXT: s_add_u32 s0, s0, s11
+; GCN-O0-NEXT: s_addc_u32 s10, s1, s10
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v42, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v41, s0
+; GCN-O0-NEXT: flat_store_dwordx4 v[41:42], v[43:46]
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v36
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v34, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[33:34], v[35:38]
+; GCN-O0-NEXT: v_mov_b32_e32 v40, v24
+; GCN-O0-NEXT: v_mov_b32_e32 v39, v23
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v22
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v21
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v20
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v19
+; GCN-O0-NEXT: v_mov_b32_e32 v34, v18
+; GCN-O0-NEXT: v_mov_b32_e32 v33, v17
+; GCN-O0-NEXT: v_mov_b32_e32 v46, v40
+; GCN-O0-NEXT: v_mov_b32_e32 v45, v39
+; GCN-O0-NEXT: v_mov_b32_e32 v44, v38
+; GCN-O0-NEXT: v_mov_b32_e32 v43, v37
+; GCN-O0-NEXT: s_mov_b32 s0, s2
+; GCN-O0-NEXT: s_mov_b32 s1, s3
+; GCN-O0-NEXT: s_mov_b32 s3, s8
+; GCN-O0-NEXT: s_mov_b32 s2, s9
+; GCN-O0-NEXT: s_add_u32 s0, s0, s3
+; GCN-O0-NEXT: s_addc_u32 s2, s1, s2
+; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GCN-O0-NEXT: s_mov_b32 s1, s2
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
-; GCN-O0-NEXT: s_mov_b32 s11, s6
-; GCN-O0-NEXT: s_mov_b32 s10, s7
-; GCN-O0-NEXT: s_add_u32 s2, s2, s11
-; GCN-O0-NEXT: s_addc_u32 s10, s3, s10
+; GCN-O0-NEXT: s_mov_b32 s9, s4
+; GCN-O0-NEXT: s_mov_b32 s8, s5
+; GCN-O0-NEXT: s_add_u32 s2, s2, s9
+; GCN-O0-NEXT: s_addc_u32 s8, s3, s8
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
-; GCN-O0-NEXT: s_mov_b32 s3, s10
-; GCN-O0-NEXT: v_mov_b32_e32 v17, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v16, s2
-; GCN-O0-NEXT: flat_store_dwordx4 v[16:17], v[18:21]
-; GCN-O0-NEXT: v_mov_b32_e32 v40, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v41, v2
-; GCN-O0-NEXT: v_mov_b32_e32 v46, v1
-; GCN-O0-NEXT: v_mov_b32_e32 v42, v0
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[56:59], 0 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v8, off, s[56:59], 0 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v9, off, s[56:59], 0 offset:36 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v10, off, s[56:59], 0 offset:40 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v11, off, s[56:59], 0 offset:44 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v12, off, s[56:59], 0 offset:48 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v13, off, s[56:59], 0 offset:52 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v14, off, s[56:59], 0 offset:56 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v15, off, s[56:59], 0 offset:60 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v16, off, s[56:59], 0 offset:64 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v17, off, s[56:59], 0 offset:68 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v18, off, s[56:59], 0 offset:72 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v19, off, s[56:59], 0 offset:76 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v20, off, s[56:59], 0 offset:80 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v21, off, s[56:59], 0 offset:84 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v22, off, s[56:59], 0 offset:88 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v23, off, s[56:59], 0 offset:92 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v24, off, s[56:59], 0 offset:96 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v25, off, s[56:59], 0 offset:100 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v26, off, s[56:59], 0 offset:104 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v27, off, s[56:59], 0 offset:108 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v28, off, s[56:59], 0 offset:112 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v29, off, s[56:59], 0 offset:116 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v30, off, s[56:59], 0 offset:120 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v31, off, s[56:59], 0 offset:124 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v32, off, s[56:59], 0 offset:128 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v33, off, s[56:59], 0 offset:132 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v34, off, s[56:59], 0 offset:136 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v35, off, s[56:59], 0 offset:140 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v36, off, s[56:59], 0 offset:144 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v37, off, s[56:59], 0 offset:148 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v38, off, s[56:59], 0 offset:152 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[56:59], 0 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v5, off, s[56:59], 0 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v4, off, s[56:59], 0 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[56:59], 0 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:8 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[56:59], 0 offset:4 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[56:59], 0 ; 4-byte Folded Reload
-; GCN-O0-NEXT: ; kill: def $vgpr42 killed $vgpr42 def $vgpr42_vgpr43_vgpr44_vgpr45 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v43, v46
-; GCN-O0-NEXT: v_mov_b32_e32 v44, v41
-; GCN-O0-NEXT: v_mov_b32_e32 v45, v40
-; GCN-O0-NEXT: v_mov_b32_e32 v41, s1
-; GCN-O0-NEXT: v_mov_b32_e32 v40, s0
-; GCN-O0-NEXT: flat_store_dwordx4 v[40:41], v[42:45]
-; GCN-O0-NEXT: ; kill: def $vgpr39 killed $vgpr39 def $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v40, v50
-; GCN-O0-NEXT: v_mov_b32_e32 v41, v49
-; GCN-O0-NEXT: v_mov_b32_e32 v42, v48
-; GCN-O0-NEXT: v_mov_b32_e32 v43, v47
-; GCN-O0-NEXT: s_waitcnt vmcnt(6)
-; GCN-O0-NEXT: v_mov_b32_e32 v44, v5
-; GCN-O0-NEXT: s_waitcnt vmcnt(5)
-; GCN-O0-NEXT: v_mov_b32_e32 v45, v4
-; GCN-O0-NEXT: s_waitcnt vmcnt(4)
-; GCN-O0-NEXT: v_mov_b32_e32 v46, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v46
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v45
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v44
-; GCN-O0-NEXT: v_mov_b32_e32 v47, v43
-; GCN-O0-NEXT: ; kill: def $vgpr47 killed $vgpr47 def $vgpr47_vgpr48_vgpr49_vgpr50 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v48, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v49, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v50, v3
+; GCN-O0-NEXT: s_mov_b32 s3, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v42, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v41, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[41:42], v[43:46]
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v36
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v35
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v34
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v33
+; GCN-O0-NEXT: v_mov_b32_e32 v34, s1
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s0
+; GCN-O0-NEXT: flat_store_dwordx4 v[33:34], v[35:38]
+; GCN-O0-NEXT: s_nop 0
+; GCN-O0-NEXT: v_mov_b32_e32 v38, v28
+; GCN-O0-NEXT: v_mov_b32_e32 v37, v27
+; GCN-O0-NEXT: v_mov_b32_e32 v36, v26
+; GCN-O0-NEXT: v_mov_b32_e32 v35, v25
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s0, s1
-; GCN-O0-NEXT: s_mov_b32 s3, s8
-; GCN-O0-NEXT: s_mov_b32 s1, s9
+; GCN-O0-NEXT: s_mov_b32 s3, s6
+; GCN-O0-NEXT: s_mov_b32 s1, s7
; GCN-O0-NEXT: s_add_u32 s2, s2, s3
; GCN-O0-NEXT: s_addc_u32 s0, s0, s1
; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b32 s3, s0
-; GCN-O0-NEXT: s_mov_b32 s0, s2
-; GCN-O0-NEXT: s_mov_b32 s1, s3
-; GCN-O0-NEXT: s_mov_b32 s9, s6
-; GCN-O0-NEXT: s_mov_b32 s8, s7
-; GCN-O0-NEXT: s_add_u32 s0, s0, s9
-; GCN-O0-NEXT: s_addc_u32 s8, s1, s8
-; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GCN-O0-NEXT: s_mov_b32 s1, s8
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s1
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s0
-; GCN-O0-NEXT: flat_store_dwordx4 v[3:4], v[47:50]
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v42
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v41
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v40
-; GCN-O0-NEXT: ; kill: def $vgpr39 killed $vgpr39 killed $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46 killed $exec
-; GCN-O0-NEXT: ; kill: def $vgpr39 killed $vgpr39 def $vgpr39_vgpr40_vgpr41_vgpr42 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v40, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v41, v4
-; GCN-O0-NEXT: v_mov_b32_e32 v42, v3
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
-; GCN-O0-NEXT: flat_store_dwordx4 v[3:4], v[39:42]
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v36
-; GCN-O0-NEXT: v_mov_b32_e32 v7, v35
-; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v8, v3
+; GCN-O0-NEXT: v_mov_b32_e32 v34, s3
+; GCN-O0-NEXT: v_mov_b32_e32 v33, s2
+; GCN-O0-NEXT: flat_store_dwordx4 v[33:34], v[35:38]
+; GCN-O0-NEXT: v_mov_b32_e32 v0, v30
+; GCN-O0-NEXT: v_mov_b32_e32 v2, v29
+; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GCN-O0-NEXT: v_mov_b32_e32 v3, v0
; GCN-O0-NEXT: s_mov_b32 s0, s2
; GCN-O0-NEXT: s_mov_b32 s1, s3
; GCN-O0-NEXT: s_mov_b32 s3, s4
@@ -3831,26 +3538,9 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: s_addc_u32 s2, s1, s2
; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GCN-O0-NEXT: s_mov_b32 s1, s2
-; GCN-O0-NEXT: s_mov_b32 s2, s0
-; GCN-O0-NEXT: s_mov_b32 s3, s1
-; GCN-O0-NEXT: s_mov_b32 s5, s6
-; GCN-O0-NEXT: s_mov_b32 s4, s7
-; GCN-O0-NEXT: s_add_u32 s2, s2, s5
-; GCN-O0-NEXT: s_addc_u32 s4, s3, s4
-; GCN-O0-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
-; GCN-O0-NEXT: s_mov_b32 s3, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s3
-; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
-; GCN-O0-NEXT: flat_store_dwordx2 v[3:4], v[7:8]
-; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: s_waitcnt vmcnt(5)
-; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: s_waitcnt vmcnt(4)
-; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
-; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GCN-O0-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-O0-NEXT: s_endpgm
entry:
%v = insertelement <15 x double> %vec, double 1.000000e+00, i32 %sel
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index a56fd4aeb980a..589948627a566 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -312,11 +312,12 @@ define amdgpu_kernel void @insertelement_v3f32_1(ptr addrspace(1) %out, <3 x flo
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s1, 0x40a00000
; SI-NEXT: s_mov_b32 s7, 0x100f000
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v1, 0x40a00000
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: buffer_store_dwordx3 v[0:2], off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -325,11 +326,12 @@ define amdgpu_kernel void @insertelement_v3f32_1(ptr addrspace(1) %out, <3 x flo
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s1, 0x40a00000
; VI-NEXT: s_mov_b32 s7, 0x1100f000
; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: v_mov_b32_e32 v1, 0x40a00000
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: buffer_store_dwordx3 v[0:2], off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -343,12 +345,13 @@ define amdgpu_kernel void @insertelement_v3f32_2(ptr addrspace(1) %out, <3 x flo
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s2, 0x40a00000
; SI-NEXT: s_mov_b32 s7, 0x100f000
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v2, 0x40a00000
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: buffer_store_dwordx3 v[0:2], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -356,12 +359,13 @@ define amdgpu_kernel void @insertelement_v3f32_2(ptr addrspace(1) %out, <3 x flo
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s2, 0x40a00000
; VI-NEXT: s_mov_b32 s7, 0x1100f000
; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: v_mov_b32_e32 v2, 0x40a00000
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: buffer_store_dwordx3 v[0:2], off, s[4:7], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <3 x float> %a, float 5.000000e+00, i32 2
@@ -654,8 +658,8 @@ define amdgpu_kernel void @dynamic_insertelement_v4f32(ptr addrspace(1) %out, <4
define amdgpu_kernel void @dynamic_insertelement_v8f32(ptr addrspace(1) %out, <8 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v8f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x8
; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x8
; SI-NEXT: s_load_dword s8, s[8:9], 0x10
; SI-NEXT: v_mov_b32_e32 v8, 0x40a00000
; SI-NEXT: s_mov_b32 s15, 0x100f000
@@ -677,8 +681,8 @@ define amdgpu_kernel void @dynamic_insertelement_v8f32(ptr addrspace(1) %out, <8
;
; VI-LABEL: dynamic_insertelement_v8f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
; VI-NEXT: s_load_dword s8, s[8:9], 0x40
; VI-NEXT: v_mov_b32_e32 v8, 0x40a00000
; VI-NEXT: s_mov_b32 s15, 0x1100f000
@@ -724,9 +728,9 @@ define amdgpu_kernel void @dynamic_insertelement_v9f32(ptr addrspace(1) %out, <9
; SI-NEXT: s_mov_b32 m0, s8
; SI-NEXT: s_mov_b32 s14, -1
; SI-NEXT: v_movreld_b32_e32 v0, v9
-; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v9f32:
@@ -751,9 +755,9 @@ define amdgpu_kernel void @dynamic_insertelement_v9f32(ptr addrspace(1) %out, <9
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, v9
-; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <9 x float> %a, float 5.000000e+00, i32 %b
store <9 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -790,29 +794,30 @@ define amdgpu_kernel void @dynamic_insertelement_v10f32(ptr addrspace(1) %out, <
;
; VI-LABEL: dynamic_insertelement_v10f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x60
-; VI-NEXT: s_load_dword s6, s[8:9], 0x80
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x60
; VI-NEXT: v_mov_b32_e32 v10, 0x40a00000
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: s_mov_b32 m0, s6
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s10
+; VI-NEXT: v_mov_b32_e32 v9, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, v10
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx2 v[8:9], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <10 x float> %a, float 5.000000e+00, i32 %b
store <10 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -822,60 +827,59 @@ define amdgpu_kernel void @dynamic_insertelement_v10f32(ptr addrspace(1) %out, <
define amdgpu_kernel void @dynamic_insertelement_v11f32(ptr addrspace(1) %out, <11 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v11f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x18
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s7, s[8:9], 0x20
+; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x18
+; SI-NEXT: s_load_dword s8, s[8:9], 0x20
; SI-NEXT: v_mov_b32_e32 v11, 0x40a00000
-; SI-NEXT: s_mov_b32 s3, 0x100f000
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_mov_b32_e32 v3, s15
-; SI-NEXT: v_mov_b32_e32 v4, s16
-; SI-NEXT: v_mov_b32_e32 v5, s17
-; SI-NEXT: v_mov_b32_e32 v6, s18
-; SI-NEXT: v_mov_b32_e32 v7, s19
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s6
+; SI-NEXT: s_mov_b32 s15, 0x100f000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 m0, s7
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v6, s6
+; SI-NEXT: v_mov_b32_e32 v7, s7
+; SI-NEXT: v_mov_b32_e32 v8, s16
+; SI-NEXT: v_mov_b32_e32 v9, s17
+; SI-NEXT: v_mov_b32_e32 v10, s18
+; SI-NEXT: s_mov_b32 m0, s8
+; SI-NEXT: s_mov_b32 s14, -1
; SI-NEXT: v_movreld_b32_e32 v0, v11
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; SI-NEXT: buffer_store_dwordx3 v[8:10], off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dwordx3 v[8:10], off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v11f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x60
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_load_dword s7, s[8:9], 0x80
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x60
; VI-NEXT: v_mov_b32_e32 v11, 0x40a00000
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: v_mov_b32_e32 v10, s6
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 m0, s7
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_mov_b32_e32 v9, s17
+; VI-NEXT: v_mov_b32_e32 v10, s18
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, v11
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx3 v[8:10], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx3 v[8:10], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <11 x float> %a, float 5.000000e+00, i32 %b
store <11 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -885,60 +889,61 @@ define amdgpu_kernel void @dynamic_insertelement_v11f32(ptr addrspace(1) %out, <
define amdgpu_kernel void @dynamic_insertelement_v12f32(ptr addrspace(1) %out, <12 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v12f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x18
+; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x18
; SI-NEXT: s_load_dword s8, s[8:9], 0x20
; SI-NEXT: v_mov_b32_e32 v12, 0x40a00000
-; SI-NEXT: s_mov_b32 s3, 0x100f000
+; SI-NEXT: s_mov_b32 s15, 0x100f000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_mov_b32_e32 v3, s15
-; SI-NEXT: v_mov_b32_e32 v4, s16
-; SI-NEXT: v_mov_b32_e32 v5, s17
-; SI-NEXT: v_mov_b32_e32 v6, s18
-; SI-NEXT: v_mov_b32_e32 v7, s19
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s6
-; SI-NEXT: v_mov_b32_e32 v11, s7
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v6, s6
+; SI-NEXT: v_mov_b32_e32 v7, s7
+; SI-NEXT: v_mov_b32_e32 v8, s16
+; SI-NEXT: v_mov_b32_e32 v9, s17
+; SI-NEXT: v_mov_b32_e32 v10, s18
+; SI-NEXT: v_mov_b32_e32 v11, s19
; SI-NEXT: s_mov_b32 m0, s8
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s14, -1
; SI-NEXT: v_movreld_b32_e32 v0, v12
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v12f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x60
-; VI-NEXT: s_load_dword s8, s[8:9], 0x80
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x60
; VI-NEXT: v_mov_b32_e32 v12, 0x40a00000
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: v_mov_b32_e32 v10, s6
-; VI-NEXT: v_mov_b32_e32 v11, s7
-; VI-NEXT: s_mov_b32 m0, s8
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_mov_b32_e32 v9, s17
+; VI-NEXT: v_mov_b32_e32 v10, s18
+; VI-NEXT: v_mov_b32_e32 v11, s19
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, v12
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <12 x float> %a, float 5.000000e+00, i32 %b
store <12 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -1155,10 +1160,10 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(ptr addrspace(1) %out, <8
; SI-LABEL: dynamic_insertelement_v8i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x8
-; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; SI-NEXT: s_load_dword s8, s[8:9], 0x10
-; SI-NEXT: s_mov_b32 s15, 0x100f000
-; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: s_load_dword s12, s[8:9], 0x10
+; SI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; SI-NEXT: s_mov_b32 s11, 0x100f000
+; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
@@ -1168,19 +1173,19 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(ptr addrspace(1) %out, <8
; SI-NEXT: v_mov_b32_e32 v5, s5
; SI-NEXT: v_mov_b32_e32 v6, s6
; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: s_mov_b32 m0, s8
+; SI-NEXT: s_mov_b32 m0, s12
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v8i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; VI-NEXT: s_load_dword s8, s[8:9], 0x40
-; VI-NEXT: s_mov_b32 s15, 0x1100f000
-; VI-NEXT: s_mov_b32 s14, -1
+; VI-NEXT: s_load_dword s12, s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT: s_mov_b32 s11, 0x1100f000
+; VI-NEXT: s_mov_b32 s10, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -1190,10 +1195,10 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(ptr addrspace(1) %out, <8
; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: v_mov_b32_e32 v6, s6
; VI-NEXT: v_mov_b32_e32 v7, s7
-; VI-NEXT: s_mov_b32 m0, s8
+; VI-NEXT: s_mov_b32 m0, s12
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <8 x i32> %a, i32 5, i32 %b
store <8 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1221,9 +1226,9 @@ define amdgpu_kernel void @dynamic_insertelement_v9i32(ptr addrspace(1) %out, <9
; SI-NEXT: v_mov_b32_e32 v8, s10
; SI-NEXT: s_mov_b32 m0, s8
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v9i32:
@@ -1247,9 +1252,9 @@ define amdgpu_kernel void @dynamic_insertelement_v9i32(ptr addrspace(1) %out, <9
; VI-NEXT: v_mov_b32_e32 v8, s0
; VI-NEXT: s_mov_b32 m0, s1
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <9 x i32> %a, i32 5, i32 %b
store <9 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1285,28 +1290,29 @@ define amdgpu_kernel void @dynamic_insertelement_v10i32(ptr addrspace(1) %out, <
;
; VI-LABEL: dynamic_insertelement_v10i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x60
-; VI-NEXT: s_load_dword s6, s[8:9], 0x80
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x60
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: s_mov_b32 m0, s6
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s10
+; VI-NEXT: v_mov_b32_e32 v9, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx2 v[8:9], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <10 x i32> %a, i32 5, i32 %b
store <10 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1316,58 +1322,57 @@ define amdgpu_kernel void @dynamic_insertelement_v10i32(ptr addrspace(1) %out, <
define amdgpu_kernel void @dynamic_insertelement_v11i32(ptr addrspace(1) %out, <11 x i32> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v11i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x18
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s7, s[8:9], 0x20
-; SI-NEXT: s_mov_b32 s3, 0x100f000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_mov_b32_e32 v3, s15
-; SI-NEXT: v_mov_b32_e32 v4, s16
-; SI-NEXT: v_mov_b32_e32 v5, s17
-; SI-NEXT: v_mov_b32_e32 v6, s18
-; SI-NEXT: v_mov_b32_e32 v7, s19
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s6
+; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x18
+; SI-NEXT: s_load_dword s8, s[8:9], 0x20
+; SI-NEXT: s_mov_b32 s15, 0x100f000
+; SI-NEXT: s_mov_b32 s14, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 m0, s7
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v6, s6
+; SI-NEXT: v_mov_b32_e32 v7, s7
+; SI-NEXT: v_mov_b32_e32 v8, s16
+; SI-NEXT: v_mov_b32_e32 v9, s17
+; SI-NEXT: v_mov_b32_e32 v10, s18
+; SI-NEXT: s_mov_b32 m0, s8
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; SI-NEXT: buffer_store_dwordx3 v[8:10], off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dwordx3 v[8:10], off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v11i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x60
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x60
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_load_dword s7, s[8:9], 0x80
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: v_mov_b32_e32 v10, s6
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_mov_b32_e32 v9, s17
+; VI-NEXT: v_mov_b32_e32 v10, s18
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 m0, s7
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx3 v[8:10], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx3 v[8:10], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <11 x i32> %a, i32 5, i32 %b
store <11 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1377,58 +1382,59 @@ define amdgpu_kernel void @dynamic_insertelement_v11i32(ptr addrspace(1) %out, <
define amdgpu_kernel void @dynamic_insertelement_v12i32(ptr addrspace(1) %out, <12 x i32> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v12i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x18
+; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x18
; SI-NEXT: s_load_dword s8, s[8:9], 0x20
-; SI-NEXT: s_mov_b32 s3, 0x100f000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s15, 0x100f000
+; SI-NEXT: s_mov_b32 s14, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_mov_b32_e32 v3, s15
-; SI-NEXT: v_mov_b32_e32 v4, s16
-; SI-NEXT: v_mov_b32_e32 v5, s17
-; SI-NEXT: v_mov_b32_e32 v6, s18
-; SI-NEXT: v_mov_b32_e32 v7, s19
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s6
-; SI-NEXT: v_mov_b32_e32 v11, s7
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v6, s6
+; SI-NEXT: v_mov_b32_e32 v7, s7
+; SI-NEXT: v_mov_b32_e32 v8, s16
+; SI-NEXT: v_mov_b32_e32 v9, s17
+; SI-NEXT: v_mov_b32_e32 v10, s18
+; SI-NEXT: v_mov_b32_e32 v11, s19
; SI-NEXT: s_mov_b32 m0, s8
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[12:15], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v12i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x60
-; VI-NEXT: s_load_dword s8, s[8:9], 0x80
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x60
+; VI-NEXT: s_mov_b32 s15, 0x1100f000
+; VI-NEXT: s_mov_b32 s14, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s16
-; VI-NEXT: v_mov_b32_e32 v5, s17
-; VI-NEXT: v_mov_b32_e32 v6, s18
-; VI-NEXT: v_mov_b32_e32 v7, s19
-; VI-NEXT: v_mov_b32_e32 v8, s4
-; VI-NEXT: v_mov_b32_e32 v9, s5
-; VI-NEXT: v_mov_b32_e32 v10, s6
-; VI-NEXT: v_mov_b32_e32 v11, s7
-; VI-NEXT: s_mov_b32 m0, s8
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_load_dword s0, s[8:9], 0x80
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_mov_b32_e32 v9, s17
+; VI-NEXT: v_mov_b32_e32 v10, s18
+; VI-NEXT: v_mov_b32_e32 v11, s19
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 m0, s0
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[12:15], 0 offset:32
; VI-NEXT: s_endpgm
%vecins = insertelement <12 x i32> %a, i32 5, i32 %b
store <12 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -2140,24 +2146,24 @@ define amdgpu_kernel void @dynamic_insertelement_v2i64(ptr addrspace(1) %out, <2
define amdgpu_kernel void @dynamic_insertelement_v3i64(ptr addrspace(1) %out, <3 x i64> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v3i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s10, s[8:9], 0x10
+; SI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0xc
+; SI-NEXT: s_load_dword s12, s[8:9], 0x10
; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x8
-; SI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0xc
; SI-NEXT: s_mov_b32 s3, 0x100f000
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s10, 1
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s12, 2
+; SI-NEXT: s_cselect_b32 s8, 0, s11
+; SI-NEXT: s_cselect_b32 s9, 5, s10
+; SI-NEXT: s_cmp_eq_u32 s12, 1
; SI-NEXT: s_cselect_b32 s7, 0, s7
; SI-NEXT: s_cselect_b32 s6, 5, s6
-; SI-NEXT: s_cmp_eq_u32 s10, 0
+; SI-NEXT: s_cmp_eq_u32 s12, 0
+; SI-NEXT: v_mov_b32_e32 v0, s9
+; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: s_cselect_b32 s5, 0, s5
; SI-NEXT: s_cselect_b32 s4, 5, s4
-; SI-NEXT: s_cmp_eq_u32 s10, 2
-; SI-NEXT: s_cselect_b32 s9, 0, s9
-; SI-NEXT: s_cselect_b32 s8, 5, s8
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 offset:16
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
@@ -2168,24 +2174,24 @@ define amdgpu_kernel void @dynamic_insertelement_v3i64(ptr addrspace(1) %out, <3
;
; VI-LABEL: dynamic_insertelement_v3i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s10, s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x30
+; VI-NEXT: s_load_dword s12, s[8:9], 0x40
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x20
-; VI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x30
; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s10, 1
; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s12, 2
+; VI-NEXT: s_cselect_b32 s8, 0, s11
+; VI-NEXT: s_cselect_b32 s9, 5, s10
+; VI-NEXT: s_cmp_eq_u32 s12, 1
; VI-NEXT: s_cselect_b32 s7, 0, s7
; VI-NEXT: s_cselect_b32 s6, 5, s6
-; VI-NEXT: s_cmp_eq_u32 s10, 0
+; VI-NEXT: s_cmp_eq_u32 s12, 0
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_mov_b32_e32 v1, s8
; VI-NEXT: s_cselect_b32 s5, 0, s5
; VI-NEXT: s_cselect_b32 s4, 5, s4
-; VI-NEXT: s_cmp_eq_u32 s10, 2
-; VI-NEXT: s_cselect_b32 s9, 0, s9
-; VI-NEXT: s_cselect_b32 s8, 5, s8
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 offset:16
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll b/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
index cd1dbb35b65d5..260bcfcc53b26 100644
--- a/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
@@ -16,8 +16,7 @@ define amdgpu_kernel void @load_idx_idy(ptr addrspace(4) %disp, ptr %g) {
; CHECK-NEXT: s_add_u32 s0, s0, s4
; CHECK-NEXT: s_addc_u32 s1, s1, s5
; CHECK-NEXT: global_load_ubyte v2, v0, s[0:1] offset:4
-; CHECK-NEXT: v_mov_b32_e32 v0, s2
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v2
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 5b95ef59499ac..f6dd8fd2d2a13 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -1107,24 +1107,23 @@ define amdgpu_kernel void @v3i32_arg(ptr addrspace(1) nocapture %out, <3 x i32>
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s2
+; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0 offset:8
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0 offset:8
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v3i32_arg:
; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_mov_b32_e32 v4, s7
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: v_mov_b32_e32 v4, s5
; VI-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
; VI-NEXT: s_endpgm
;
@@ -1182,24 +1181,23 @@ define amdgpu_kernel void @v3f32_arg(ptr addrspace(1) nocapture %out, <3 x float
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s2
+; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0 offset:8
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0 offset:8
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v3f32_arg:
; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_mov_b32_e32 v4, s7
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: v_mov_b32_e32 v4, s5
; VI-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
; VI-NEXT: s_endpgm
;
@@ -2012,34 +2010,33 @@ define amdgpu_kernel void @v5f32_arg(ptr addrspace(1) nocapture %out, <5 x float
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s4, s6, 16
; VI-NEXT: s_addc_u32 s5, s7, 0
-; VI-NEXT: v_mov_b32_e32 v3, s8
-; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v4, s6
+; VI-NEXT: v_mov_b32_e32 v5, s7
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: flat_store_dword v[1:2], v3
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: v_mov_b32_e32 v4, s6
-; VI-NEXT: v_mov_b32_e32 v5, s7
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v5f32_arg:
; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dword s6, s[8:9], 0x30
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x20
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GFX9-NEXT: s_load_dword s6, s[8:9], 0x30
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: global_store_dword v4, v5, s[4:5] offset:16
; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: global_store_dword v4, v0, s[4:5] offset:16
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v5f32_arg:
@@ -2862,55 +2859,53 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32>
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v8i32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x44
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: s_add_u32 s2, s0, 16
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_add_u32 s10, s8, 16
+; VI-NEXT: s_addc_u32 s11, s9, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v10, s10
+; VI-NEXT: v_mov_b32_e32 v11, s11
+; VI-NEXT: v_mov_b32_e32 v8, s8
+; VI-NEXT: v_mov_b32_e32 v9, s9
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v6, s2
; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v8i32_arg:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v7, s3
+; GFX9-NEXT: v_mov_b32_e32 v6, s2
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[10:11] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[10:11]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v8i32_arg:
@@ -2969,55 +2964,53 @@ define amdgpu_kernel void @v8f32_arg(ptr addrspace(1) nocapture %out, <8 x float
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v8f32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x44
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: s_add_u32 s2, s0, 16
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_add_u32 s10, s8, 16
+; VI-NEXT: s_addc_u32 s11, s9, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v10, s10
+; VI-NEXT: v_mov_b32_e32 v11, s11
+; VI-NEXT: v_mov_b32_e32 v8, s8
+; VI-NEXT: v_mov_b32_e32 v9, s9
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v6, s2
; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v8f32_arg:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v7, s3
+; GFX9-NEXT: v_mov_b32_e32 v6, s2
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[10:11] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[10:11]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v8f32_arg:
@@ -3527,55 +3520,53 @@ define amdgpu_kernel void @v16i16_arg(ptr addrspace(1) %out, <16 x i16> %in) {
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v16i16_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x44
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s12
-; VI-NEXT: s_add_u32 s2, s0, 16
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_add_u32 s10, s8, 16
+; VI-NEXT: s_addc_u32 s11, s9, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v10, s10
+; VI-NEXT: v_mov_b32_e32 v11, s11
+; VI-NEXT: v_mov_b32_e32 v8, s8
+; VI-NEXT: v_mov_b32_e32 v9, s9
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v6, s2
; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v16i16_arg:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v7, s3
+; GFX9-NEXT: v_mov_b32_e32 v6, s2
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[10:11] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[10:11]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v16i16_arg:
@@ -3980,14 +3971,13 @@ define amdgpu_kernel void @v16i32_arg(ptr addrspace(1) nocapture %out, <16 x i32
; SI-NEXT: v_mov_b32_e32 v1, s21
; SI-NEXT: v_mov_b32_e32 v2, s22
; SI-NEXT: v_mov_b32_e32 v3, s23
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s16
-; SI-NEXT: v_mov_b32_e32 v1, s17
-; SI-NEXT: v_mov_b32_e32 v2, s18
-; SI-NEXT: v_mov_b32_e32 v3, s19
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
+; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mov_b32_e32 v0, s12
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_mov_b32_e32 v2, s14
@@ -4003,42 +3993,42 @@ define amdgpu_kernel void @v16i32_arg(ptr addrspace(1) nocapture %out, <16 x i32
;
; VI-LABEL: v16i32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x64
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s20
-; VI-NEXT: s_add_u32 s2, s0, 48
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: s_add_u32 s2, s0, 32
-; VI-NEXT: v_mov_b32_e32 v1, s21
-; VI-NEXT: v_mov_b32_e32 v2, s22
-; VI-NEXT: v_mov_b32_e32 v3, s23
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: s_add_u32 s2, s0, 16
-; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: v_mov_b32_e32 v1, s17
-; VI-NEXT: v_mov_b32_e32 v2, s18
-; VI-NEXT: v_mov_b32_e32 v3, s19
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s3
+; VI-NEXT: s_add_u32 s18, s16, 48
+; VI-NEXT: s_addc_u32 s19, s17, 0
; VI-NEXT: v_mov_b32_e32 v0, s12
; VI-NEXT: v_mov_b32_e32 v1, s13
; VI-NEXT: v_mov_b32_e32 v2, s14
; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s2
+; VI-NEXT: s_add_u32 s12, s16, 32
+; VI-NEXT: v_mov_b32_e32 v4, s18
+; VI-NEXT: v_mov_b32_e32 v5, s19
+; VI-NEXT: s_addc_u32 s13, s17, 0
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s12
; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_mov_b32_e32 v2, s10
; VI-NEXT: v_mov_b32_e32 v3, s11
-; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: s_add_u32 s8, s16, 16
+; VI-NEXT: v_mov_b32_e32 v5, s13
+; VI-NEXT: s_addc_u32 s9, s17, 0
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: v_mov_b32_e32 v4, s8
+; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
;
@@ -4046,31 +4036,29 @@ define amdgpu_kernel void @v16i32_arg(ptr addrspace(1) nocapture %out, <16 x i32
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx16 s[12:27], s[8:9], 0x40
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s24
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_mov_b32_e32 v2, s26
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s20
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_mov_b32_e32 v2, s22
-; GFX9-NEXT: v_mov_b32_e32 v3, s23
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s20
+; GFX9-NEXT: v_mov_b32_e32 v5, s21
+; GFX9-NEXT: v_mov_b32_e32 v6, s22
+; GFX9-NEXT: v_mov_b32_e32 v7, s23
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:32
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_mov_b32_e32 v0, s12
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_mov_b32_e32 v2, s14
; GFX9-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v16i32_arg:
@@ -4157,14 +4145,13 @@ define amdgpu_kernel void @v16f32_arg(ptr addrspace(1) nocapture %out, <16 x flo
; SI-NEXT: v_mov_b32_e32 v1, s21
; SI-NEXT: v_mov_b32_e32 v2, s22
; SI-NEXT: v_mov_b32_e32 v3, s23
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s16
-; SI-NEXT: v_mov_b32_e32 v1, s17
-; SI-NEXT: v_mov_b32_e32 v2, s18
-; SI-NEXT: v_mov_b32_e32 v3, s19
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
+; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mov_b32_e32 v0, s12
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_mov_b32_e32 v2, s14
@@ -4180,42 +4167,42 @@ define amdgpu_kernel void @v16f32_arg(ptr addrspace(1) nocapture %out, <16 x flo
;
; VI-LABEL: v16f32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x64
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s20
-; VI-NEXT: s_add_u32 s2, s0, 48
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: s_add_u32 s2, s0, 32
-; VI-NEXT: v_mov_b32_e32 v1, s21
-; VI-NEXT: v_mov_b32_e32 v2, s22
-; VI-NEXT: v_mov_b32_e32 v3, s23
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: s_add_u32 s2, s0, 16
-; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: v_mov_b32_e32 v1, s17
-; VI-NEXT: v_mov_b32_e32 v2, s18
-; VI-NEXT: v_mov_b32_e32 v3, s19
-; VI-NEXT: s_addc_u32 s3, s1, 0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s3
+; VI-NEXT: s_add_u32 s18, s16, 48
+; VI-NEXT: s_addc_u32 s19, s17, 0
; VI-NEXT: v_mov_b32_e32 v0, s12
; VI-NEXT: v_mov_b32_e32 v1, s13
; VI-NEXT: v_mov_b32_e32 v2, s14
; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s2
+; VI-NEXT: s_add_u32 s12, s16, 32
+; VI-NEXT: v_mov_b32_e32 v4, s18
+; VI-NEXT: v_mov_b32_e32 v5, s19
+; VI-NEXT: s_addc_u32 s13, s17, 0
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s12
; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_mov_b32_e32 v2, s10
; VI-NEXT: v_mov_b32_e32 v3, s11
-; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: s_add_u32 s8, s16, 16
+; VI-NEXT: v_mov_b32_e32 v5, s13
+; VI-NEXT: s_addc_u32 s9, s17, 0
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: v_mov_b32_e32 v4, s8
+; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
;
@@ -4223,31 +4210,29 @@ define amdgpu_kernel void @v16f32_arg(ptr addrspace(1) nocapture %out, <16 x flo
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx16 s[12:27], s[8:9], 0x40
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s24
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_mov_b32_e32 v2, s26
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s20
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_mov_b32_e32 v2, s22
-; GFX9-NEXT: v_mov_b32_e32 v3, s23
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s20
+; GFX9-NEXT: v_mov_b32_e32 v5, s21
+; GFX9-NEXT: v_mov_b32_e32 v6, s22
+; GFX9-NEXT: v_mov_b32_e32 v7, s23
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:32
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_mov_b32_e32 v0, s12
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_mov_b32_e32 v2, s14
; GFX9-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v16f32_arg:
@@ -5942,16 +5927,16 @@ define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(ptr addrspace
;
; VI-LABEL: byref_natural_align_constant_v16i32_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VI-NEXT: s_load_dword s4, s[4:5], 0xa4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s20
; VI-NEXT: s_add_u32 s2, s0, 48
; VI-NEXT: s_addc_u32 s3, s1, 0
; VI-NEXT: v_mov_b32_e32 v5, s3
; VI-NEXT: v_mov_b32_e32 v4, s2
; VI-NEXT: s_add_u32 s2, s0, 32
+; VI-NEXT: v_mov_b32_e32 v0, s20
; VI-NEXT: v_mov_b32_e32 v1, s21
; VI-NEXT: v_mov_b32_e32 v2, s22
; VI-NEXT: v_mov_b32_e32 v3, s23
@@ -5969,19 +5954,19 @@ define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(ptr addrspace
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v5, s3
+; VI-NEXT: v_mov_b32_e32 v4, s2
; VI-NEXT: v_mov_b32_e32 v0, s12
; VI-NEXT: v_mov_b32_e32 v1, s13
; VI-NEXT: v_mov_b32_e32 v2, s14
; VI-NEXT: v_mov_b32_e32 v3, s15
-; VI-NEXT: v_mov_b32_e32 v4, s2
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_mov_b32_e32 v2, s10
; VI-NEXT: v_mov_b32_e32 v3, s11
-; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -5994,34 +5979,34 @@ define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(ptr addrspace
; GFX9-NEXT: s_load_dwordx16 s[12:27], s[8:9], 0x40
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT: s_load_dword s2, s[8:9], 0x80
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s24
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_mov_b32_e32 v2, s26
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; GFX9-NEXT: v_mov_b32_e32 v4, s20
+; GFX9-NEXT: v_mov_b32_e32 v5, s21
+; GFX9-NEXT: v_mov_b32_e32 v6, s22
+; GFX9-NEXT: v_mov_b32_e32 v7, s23
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s20
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_mov_b32_e32 v2, s22
-; GFX9-NEXT: v_mov_b32_e32 v3, s23
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:32
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s12
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_mov_b32_e32 v2, s14
; GFX9-NEXT: v_mov_b32_e32 v3, s15
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v4, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v8, v0, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
index e9f4ccda47905..9e47a6197fa3a 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
@@ -527,36 +527,36 @@ define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(ptr addrspace
; GCN-LABEL: byref_natural_align_constant_v16i32_arg:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx16 s[12:27], s[8:9], 0x40
-; GCN-NEXT: s_load_dword s2, s[8:9], 0x80
; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: v_mov_b32_e32 v4, 0
+; GCN-NEXT: s_load_dword s2, s[8:9], 0x80
+; GCN-NEXT: v_mov_b32_e32 v8, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s24
; GCN-NEXT: v_mov_b32_e32 v1, s25
; GCN-NEXT: v_mov_b32_e32 v2, s26
; GCN-NEXT: v_mov_b32_e32 v3, s27
-; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; GCN-NEXT: v_mov_b32_e32 v4, s20
+; GCN-NEXT: v_mov_b32_e32 v5, s21
+; GCN-NEXT: v_mov_b32_e32 v6, s22
+; GCN-NEXT: v_mov_b32_e32 v7, s23
+; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s20
-; GCN-NEXT: v_mov_b32_e32 v1, s21
-; GCN-NEXT: v_mov_b32_e32 v2, s22
-; GCN-NEXT: v_mov_b32_e32 v3, s23
-; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; GCN-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:32
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s16
; GCN-NEXT: v_mov_b32_e32 v1, s17
; GCN-NEXT: v_mov_b32_e32 v2, s18
; GCN-NEXT: v_mov_b32_e32 v3, s19
-; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s12
; GCN-NEXT: v_mov_b32_e32 v1, s13
; GCN-NEXT: v_mov_b32_e32 v2, s14
; GCN-NEXT: v_mov_b32_e32 v3, s15
-; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: global_store_dword v4, v0, s[0:1]
+; GCN-NEXT: global_store_dword v8, v0, s[0:1]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
%in = load <16 x i32>, ptr addrspace(4) %in.byref
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
index 3d702b67835ad..688733228e067 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
@@ -6,25 +6,25 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: ds_read_b128 v[30:33], v0 offset:112
-; GCN-NEXT: ds_read_b128 v[26:29], v0 offset:96
-; GCN-NEXT: ds_read_b128 v[22:25], v0 offset:80
-; GCN-NEXT: ds_read_b128 v[18:21], v0 offset:64
+; GCN-NEXT: ds_read_b128 v[22:25], v0 offset:112
+; GCN-NEXT: ds_read_b128 v[18:21], v0 offset:96
+; GCN-NEXT: ds_read_b128 v[14:17], v0 offset:80
+; GCN-NEXT: ds_read_b128 v[10:13], v0 offset:64
; GCN-NEXT: ds_read_b128 v[2:5], v0
; GCN-NEXT: ds_read_b128 v[6:9], v0 offset:16
-; GCN-NEXT: ds_read_b128 v[10:13], v0 offset:32
-; GCN-NEXT: ds_read_b128 v[14:17], v0 offset:48
+; GCN-NEXT: ds_read_b128 v[26:29], v0 offset:32
+; GCN-NEXT: ds_read_b128 v[30:33], v0 offset:48
; GCN-NEXT: v_mov_b32_e32 v34, 0
; GCN-NEXT: v_mov_b32_e32 v35, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u64 s[0:1], 0
; GCN-NEXT: ; iglp_opt mask(0x00000001)
-; GCN-NEXT: ds_write_b128 v0, v[30:33] offset:112
-; GCN-NEXT: ds_write_b128 v0, v[26:29] offset:96
-; GCN-NEXT: ds_write_b128 v0, v[22:25] offset:80
-; GCN-NEXT: ds_write_b128 v0, v[18:21] offset:64
-; GCN-NEXT: ds_write_b128 v0, v[14:17] offset:48
-; GCN-NEXT: ds_write_b128 v0, v[10:13] offset:32
+; GCN-NEXT: ds_write_b128 v0, v[22:25] offset:112
+; GCN-NEXT: ds_write_b128 v0, v[18:21] offset:96
+; GCN-NEXT: ds_write_b128 v0, v[14:17] offset:80
+; GCN-NEXT: ds_write_b128 v0, v[10:13] offset:64
+; GCN-NEXT: ds_write_b128 v0, v[30:33] offset:48
+; GCN-NEXT: ds_write_b128 v0, v[26:29] offset:32
; GCN-NEXT: ds_write_b128 v0, v[6:9] offset:16
; GCN-NEXT: ds_write_b128 v0, v[2:5]
; GCN-NEXT: ds_write_b64 v0, v[34:35]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index 5810d859f0cf4..e6cdb2ae87801 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -32,86 +32,86 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; SDAG-NEXT: ds_read_b128 a[8:11], v2 offset:57376
; SDAG-NEXT: ds_read_b128 a[12:15], v2 offset:57392
; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
-; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:49264
-; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:49248
-; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:49232
-; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:49216
-; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:49200
-; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:49184
-; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:49168
-; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:49152
+; SDAG-NEXT: ds_read_b128 a[156:159], v1 offset:49264
+; SDAG-NEXT: ds_read_b128 a[152:155], v1 offset:49248
+; SDAG-NEXT: ds_read_b128 a[148:151], v1 offset:49232
+; SDAG-NEXT: ds_read_b128 a[144:147], v1 offset:49216
+; SDAG-NEXT: ds_read_b128 a[140:143], v1 offset:49200
+; SDAG-NEXT: ds_read_b128 a[136:139], v1 offset:49184
+; SDAG-NEXT: ds_read_b128 a[132:135], v1 offset:49168
+; SDAG-NEXT: ds_read_b128 a[128:131], v1 offset:49152
; SDAG-NEXT: s_waitcnt lgkmcnt(8)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; SDAG-NEXT: ds_read_b128 a[156:159], v1 offset:112
-; SDAG-NEXT: ds_read_b128 a[152:155], v1 offset:96
-; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:24576
; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
; SDAG-NEXT: s_waitcnt lgkmcnt(4)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
-; SDAG-NEXT: ds_read_b128 a[148:151], v1 offset:80
-; SDAG-NEXT: ds_read_b128 a[144:147], v1 offset:64
-; SDAG-NEXT: ds_read_b128 a[128:131], v1
-; SDAG-NEXT: ds_read_b128 a[132:135], v1 offset:16
-; SDAG-NEXT: ds_read_b128 a[136:139], v1 offset:32
-; SDAG-NEXT: ds_read_b128 a[140:143], v1 offset:48
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159]
-; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:8304
-; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:8288
-; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:8272
-; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:8256
-; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:8240
-; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:8224
-; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:8208
-; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:8192
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[32:35], v1
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
-; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:8304
+; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
+; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:24608
; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:112
-; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[128:131]
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[32:35]
; SDAG-NEXT: v_mov_b32_e32 v0, s1
; SDAG-NEXT: s_waitcnt lgkmcnt(8)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:24672
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:24688
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:24640
-; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:8208
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:24656
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:24608
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:24624
-; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:24576
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:24592
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:24592
; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:16400
; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
@@ -437,49 +437,49 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:48
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
-; SDAG-NEXT: ds_read_b128 a[156:159], v3 offset:8304
-; SDAG-NEXT: ds_read_b128 a[152:155], v3 offset:8288
-; SDAG-NEXT: ds_read_b128 a[148:151], v3 offset:8272
-; SDAG-NEXT: ds_read_b128 a[144:147], v3 offset:8256
-; SDAG-NEXT: ds_read_b128 a[140:143], v3 offset:8240
-; SDAG-NEXT: ds_read_b128 a[136:139], v3 offset:8224
-; SDAG-NEXT: ds_read_b128 a[132:135], v3 offset:8208
-; SDAG-NEXT: ds_read_b128 a[128:131], v3 offset:8192
+; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; SDAG-NEXT: ds_read_b128 a[32:35], v3 offset:8192
; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3
; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
; SDAG-NEXT: ; iglp_opt mask(0x00000001)
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
-; SDAG-NEXT: ds_read_b128 a[124:127], v3 offset:24688
-; SDAG-NEXT: ds_read_b128 a[120:123], v3 offset:24672
-; SDAG-NEXT: ds_read_b128 a[116:119], v3 offset:24656
-; SDAG-NEXT: ds_read_b128 a[112:115], v3 offset:24640
-; SDAG-NEXT: ds_read_b128 a[108:111], v3 offset:24624
-; SDAG-NEXT: ds_read_b128 a[104:107], v3 offset:24608
-; SDAG-NEXT: ds_read_b128 a[100:103], v3 offset:24592
-; SDAG-NEXT: ds_read_b128 a[96:99], v3 offset:24576
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
-; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:49264
-; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:49248
-; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:49232
-; SDAG-NEXT: ds_read_b128 a[80:83], v3 offset:49216
-; SDAG-NEXT: ds_read_b128 a[76:79], v3 offset:49200
-; SDAG-NEXT: ds_read_b128 a[72:75], v3 offset:49184
-; SDAG-NEXT: ds_read_b128 a[68:71], v3 offset:49168
-; SDAG-NEXT: ds_read_b128 a[64:67], v3 offset:49152
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:24688
+; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:24672
+; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:24656
+; SDAG-NEXT: ds_read_b128 a[80:83], v3 offset:24640
+; SDAG-NEXT: ds_read_b128 a[76:79], v3 offset:24624
+; SDAG-NEXT: ds_read_b128 a[72:75], v3 offset:24608
+; SDAG-NEXT: ds_read_b128 a[68:71], v3 offset:24592
+; SDAG-NEXT: ds_read_b128 a[64:67], v3 offset:24576
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; SDAG-NEXT: ds_read_b128 a[60:63], v4 offset:57456
-; SDAG-NEXT: ds_read_b128 a[56:59], v4 offset:57440
-; SDAG-NEXT: ds_read_b128 a[52:55], v4 offset:57424
-; SDAG-NEXT: ds_read_b128 a[48:51], v4 offset:57408
-; SDAG-NEXT: ds_read_b128 a[32:35], v4 offset:57344
-; SDAG-NEXT: ds_read_b128 a[36:39], v4 offset:57360
-; SDAG-NEXT: ds_read_b128 a[40:43], v4 offset:57376
-; SDAG-NEXT: ds_read_b128 a[44:47], v4 offset:57392
+; SDAG-NEXT: ds_read_b128 a[124:127], v3 offset:49264
+; SDAG-NEXT: ds_read_b128 a[120:123], v3 offset:49248
+; SDAG-NEXT: ds_read_b128 a[116:119], v3 offset:49232
+; SDAG-NEXT: ds_read_b128 a[112:115], v3 offset:49216
+; SDAG-NEXT: ds_read_b128 a[108:111], v3 offset:49200
+; SDAG-NEXT: ds_read_b128 a[104:107], v3 offset:49184
+; SDAG-NEXT: ds_read_b128 a[100:103], v3 offset:49168
+; SDAG-NEXT: ds_read_b128 a[96:99], v3 offset:49152
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
+; SDAG-NEXT: ds_read_b128 a[156:159], v4 offset:57456
+; SDAG-NEXT: ds_read_b128 a[152:155], v4 offset:57440
+; SDAG-NEXT: ds_read_b128 a[148:151], v4 offset:57424
+; SDAG-NEXT: ds_read_b128 a[144:147], v4 offset:57408
+; SDAG-NEXT: ds_read_b128 a[128:131], v4 offset:57344
+; SDAG-NEXT: ds_read_b128 a[132:135], v4 offset:57360
+; SDAG-NEXT: ds_read_b128 a[136:139], v4 offset:57376
+; SDAG-NEXT: ds_read_b128 a[140:143], v4 offset:57392
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
@@ -489,38 +489,38 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
; SDAG-NEXT: ds_write_b128 v0, a[0:3]
; SDAG-NEXT: v_mov_b32_e32 v0, s1
-; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:8208
-; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:16480
-; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:16496
-; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:16448
-; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:16464
-; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:16416
-; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:16432
-; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:16384
-; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:16400
-; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:24672
-; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:24688
-; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:24640
-; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:24656
-; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:24608
-; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:24624
-; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:24576
-; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:24592
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:32864
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:32880
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:32832
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:32848
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32800
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:32816
-; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:32768
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:32784
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:24592
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:32784
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_iglp_opt_rev_mfma_gemm:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index 3e9c6f4d24d83..343c8a48ac558 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -376,9 +376,9 @@ define amdgpu_ps <4 x float> @image_sample_b_2d_v3f16_tfe(<8 x i32> inreg %rsrc,
; TONGA-NEXT: v_mov_b32_e32 v6, v3
; TONGA-NEXT: s_and_b64 exec, exec, s[12:13]
; TONGA-NEXT: image_sample_b v[3:6], v[0:2], s[0:7], s[8:11] dmask:0x7 tfe d16
-; TONGA-NEXT: s_mov_b32 s0, 0x1000504
; TONGA-NEXT: s_waitcnt vmcnt(0)
-; TONGA-NEXT: v_perm_b32 v0, v3, v4, s0
+; TONGA-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; TONGA-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; TONGA-NEXT: v_mov_b32_e32 v1, v5
; TONGA-NEXT: v_mov_b32_e32 v2, v6
; TONGA-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
index 279de9b7899e0..09f1648fc3e44 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
@@ -92,59 +92,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; GFX908-SDAG-NEXT: s_nop 15
; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; GFX908-SDAG-NEXT: s_endpgm
@@ -394,67 +394,66 @@ define amdgpu_kernel void @test_mfma_f32_16x16x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG: ; %bb.0: ; %bb
; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v12, 0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v13
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v13
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s6
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v13
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s9
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v13
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s12
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v13
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s15
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v13
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2
; GFX908-SDAG-NEXT: s_nop 1
; GFX908-SDAG-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; GFX908-SDAG-NEXT: s_nop 9
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4
-; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a0
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; GFX908-SDAG-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_16x16x2bf16:
@@ -710,14 +709,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; GFX908-SDAG-NEXT: s_nop 15
; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
index 4671993a13f0d..95e24ce25db3f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
@@ -1339,9 +1339,8 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX90A-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v8, s10
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
-; GFX90A-NEXT: v_mov_b32_e32 v9, s11
; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
@@ -1384,9 +1383,8 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, s10
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
; GFX942-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v9, s11
; GFX942-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
@@ -1429,9 +1427,8 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, s10
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v9, s11
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
@@ -1474,9 +1471,8 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX942-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s10
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
; GFX942-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s11
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
@@ -2985,8 +2981,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-NEXT: v_mov_b32_e32 v7, 0x3ff00000
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v10, s2
-; GFX90A-NEXT: v_mov_b32_e32 v11, s3
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, v0
@@ -3037,8 +3032,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-NEXT: v_mov_b32_e32 v7, 0x3ff00000
; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, s2
-; GFX942-NEXT: v_mov_b32_e32 v11, s3
+; GFX942-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v2, v0
; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v4, v0
@@ -3089,8 +3083,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s3
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0
@@ -3141,8 +3134,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000
; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s2
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s3
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0
; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0
@@ -3199,8 +3191,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-NEXT: v_mov_b32_e32 v1, 0x405ec000
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v10, s2
-; GFX90A-NEXT: v_mov_b32_e32 v11, s3
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: v_mov_b32_e32 v4, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
@@ -3252,8 +3243,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-NEXT: v_mov_b32_e32 v1, 0x405ec000
; GFX942-NEXT: v_mov_b32_e32 v2, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, s2
-; GFX942-NEXT: v_mov_b32_e32 v11, s3
+; GFX942-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: v_mov_b32_e32 v4, v0
; GFX942-NEXT: v_mov_b32_e32 v5, v1
@@ -3305,8 +3295,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s3
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v1
@@ -3358,8 +3347,7 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000
; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s2
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s3
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1
; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
index 97aa940769b34..4d477110b161c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
@@ -2051,9 +2051,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2
; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2098,9 +2097,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2
; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2261,9 +2259,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -2326,9 +2323,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -2565,9 +2561,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2612,9 +2607,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2776,9 +2770,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2823,9 +2816,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -2987,9 +2979,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -3034,9 +3025,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -3198,9 +3188,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -3245,9 +3234,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
@@ -3408,9 +3396,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -3473,9 +3460,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -3711,9 +3697,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -3776,9 +3761,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -4014,9 +3998,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -4079,9 +4062,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -4317,9 +4299,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
@@ -4382,9 +4363,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[16:17]
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..5802b08febfec 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -106,16 +106,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: s_nop 4
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; SDAG-AGPR-NEXT: s_nop 6
; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -126,19 +124,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
@@ -223,8 +214,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -239,12 +230,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v48, s16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-VGPR-NEXT: s_nop 8
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-VGPR-NEXT: s_nop 1
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-VGPR-NEXT: s_nop 2
; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -255,24 +251,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
@@ -429,16 +411,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: s_nop 4
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; SDAG-AGPR-NEXT: s_nop 6
; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -449,19 +429,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
@@ -546,8 +519,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -562,12 +535,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v48, s16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-VGPR-NEXT: s_nop 8
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-VGPR-NEXT: s_nop 1
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-VGPR-NEXT: s_nop 2
; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -578,24 +556,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
@@ -939,49 +903,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[34:37], v[38:41], v[0:15]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 2
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
;
@@ -1042,49 +995,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[34:37], v[38:41], v[0:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
@@ -1208,49 +1150,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 2
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
;
@@ -1311,49 +1242,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 42cd877c42261..f54cac6399946 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -363,8 +363,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -379,12 +379,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: s_nop 8
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -395,24 +400,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
@@ -497,16 +488,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15]
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: s_nop 4
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -517,19 +506,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v0, s8
-; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v0, s12
-; HEURRC-NEXT: v_mov_b32_e32 v1, s13
-; HEURRC-NEXT: v_mov_b32_e32 v2, s14
-; HEURRC-NEXT: v_mov_b32_e32 v3, s15
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
@@ -539,8 +521,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -555,12 +537,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: s_nop 8
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -571,24 +558,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
@@ -729,8 +702,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -745,12 +718,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: s_nop 8
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -761,24 +739,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
@@ -863,16 +827,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: s_nop 4
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -883,19 +845,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v0, s8
-; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v0, s12
-; HEURRC-NEXT: v_mov_b32_e32 v1, s13
-; HEURRC-NEXT: v_mov_b32_e32 v2, s14
-; HEURRC-NEXT: v_mov_b32_e32 v3, s15
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
@@ -905,8 +860,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
@@ -921,12 +876,17 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: s_nop 8
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -937,24 +897,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
@@ -1407,49 +1353,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15]
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1510,49 +1445,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 2
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s12
-; HEURRC-NEXT: v_mov_b32_e32 v17, s13
-; HEURRC-NEXT: v_mov_b32_e32 v18, s14
-; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s8
-; HEURRC-NEXT: v_mov_b32_e32 v17, s9
-; HEURRC-NEXT: v_mov_b32_e32 v18, s10
-; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1567,49 +1491,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 2
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
@@ -1749,49 +1662,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1852,49 +1754,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 2
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s12
-; HEURRC-NEXT: v_mov_b32_e32 v17, s13
-; HEURRC-NEXT: v_mov_b32_e32 v18, s14
-; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s8
-; HEURRC-NEXT: v_mov_b32_e32 v17, s9
-; HEURRC-NEXT: v_mov_b32_e32 v18, s10
-; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1909,49 +1800,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 2
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
@@ -2567,10 +2447,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; SDAG-NEXT: v_mov_b32_e32 v11, s13
; SDAG-NEXT: v_mov_b32_e32 v12, s14
; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
; SDAG-NEXT: s_nop 7
@@ -2611,10 +2489,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; HEURRC-NEXT: v_mov_b32_e32 v11, s13
; HEURRC-NEXT: v_mov_b32_e32 v12, s14
; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
; HEURRC-NEXT: s_nop 7
@@ -2636,10 +2512,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
; VGPRRC-NEXT: s_nop 7
@@ -2714,10 +2588,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; SDAG-NEXT: v_mov_b32_e32 v11, s13
; SDAG-NEXT: v_mov_b32_e32 v12, s14
; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: s_nop 7
@@ -2758,10 +2630,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; HEURRC-NEXT: v_mov_b32_e32 v11, s13
; HEURRC-NEXT: v_mov_b32_e32 v12, s14
; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
@@ -2783,10 +2653,8 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
@@ -2857,17 +2725,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -2877,9 +2741,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-NEXT: s_nop 11
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -2888,31 +2760,12 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s16
-; SDAG-NEXT: v_mov_b32_e32 v1, s17
-; SDAG-NEXT: v_mov_b32_e32 v2, s18
-; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
@@ -2976,14 +2829,10 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
@@ -3001,13 +2850,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 0
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15]
-; HEURRC-NEXT: v_mov_b32_e32 v8, s16
-; HEURRC-NEXT: v_mov_b32_e32 v9, s17
-; HEURRC-NEXT: v_mov_b32_e32 v10, s18
-; HEURRC-NEXT: v_mov_b32_e32 v11, s19
-; HEURRC-NEXT: s_nop 7
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -3016,25 +2866,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v0, s8
-; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v0, s12
-; HEURRC-NEXT: v_mov_b32_e32 v1, s13
-; HEURRC-NEXT: v_mov_b32_e32 v2, s14
-; HEURRC-NEXT: v_mov_b32_e32 v3, s15
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
@@ -3044,17 +2883,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3064,9 +2899,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: s_nop 0
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPRRC-NEXT: s_nop 11
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -3075,31 +2918,12 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
@@ -3252,17 +3076,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3272,9 +3092,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: s_nop 11
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -3283,31 +3111,12 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s16
-; SDAG-NEXT: v_mov_b32_e32 v1, s17
-; SDAG-NEXT: v_mov_b32_e32 v2, s18
-; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
@@ -3371,14 +3180,10 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
@@ -3396,13 +3201,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 0
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; HEURRC-NEXT: v_mov_b32_e32 v8, s16
-; HEURRC-NEXT: v_mov_b32_e32 v9, s17
-; HEURRC-NEXT: v_mov_b32_e32 v10, s18
-; HEURRC-NEXT: v_mov_b32_e32 v11, s19
-; HEURRC-NEXT: s_nop 7
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
@@ -3411,25 +3217,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v0, s8
-; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v0, s12
-; HEURRC-NEXT: v_mov_b32_e32 v1, s13
-; HEURRC-NEXT: v_mov_b32_e32 v2, s14
-; HEURRC-NEXT: v_mov_b32_e32 v3, s15
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
@@ -3439,17 +3234,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3459,9 +3250,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: s_nop 0
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: s_nop 11
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[58:59], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
@@ -3470,31 +3269,12 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[52:55], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[56:59], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
@@ -3950,65 +3730,47 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v32, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-NEXT: s_nop 6
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15]
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4060,129 +3822,93 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v32, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; HEURRC-NEXT: s_nop 6
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s12
-; HEURRC-NEXT: v_mov_b32_e32 v17, s13
-; HEURRC-NEXT: v_mov_b32_e32 v18, s14
-; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s8
-; HEURRC-NEXT: v_mov_b32_e32 v17, s9
-; HEURRC-NEXT: v_mov_b32_e32 v18, s10
-; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPRRC-NEXT: s_nop 6
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
@@ -4327,65 +4053,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v32, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-NEXT: s_nop 6
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4437,129 +4145,93 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v32, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; HEURRC-NEXT: s_nop 6
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; HEURRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; HEURRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; HEURRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; HEURRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; HEURRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s12
-; HEURRC-NEXT: v_mov_b32_e32 v17, s13
-; HEURRC-NEXT: v_mov_b32_e32 v18, s14
-; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s8
-; HEURRC-NEXT: v_mov_b32_e32 v17, s9
-; HEURRC-NEXT: v_mov_b32_e32 v18, s10
-; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPRRC-NEXT: s_nop 6
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[20:21]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[34:37], v[38:41], v[0:15] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[52:53], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[56:57], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[54:55], s[8:9]
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[46:49], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[50:53], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[54:57], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
@@ -4704,19 +4376,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4765,19 +4432,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4798,19 +4460,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4910,19 +4567,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4971,19 +4623,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -5004,19 +4651,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 49d4662300f76..eb169282b6ced 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -57,27 +57,26 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; GFX908-SDAG-NEXT: s_nop 15
; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a15
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a14
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a13
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a12
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a7
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a6
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a5
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a4
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a11
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a10
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a9
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a8
-; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a0
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; GFX908-SDAG-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_i32_32x32x8i8:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 086fc6f0ed4d7..32a3707e5d701 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -104,59 +104,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
@@ -359,59 +359,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
@@ -715,67 +715,66 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v12, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s4
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s6
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s7
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s8
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s9
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s10
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s11
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s12
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s14
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s15
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32:
@@ -848,67 +847,66 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v12, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s4
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s6
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s7
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s8
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s9
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s10
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s11
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s12
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s14
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s15
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_16x16x1f32:
@@ -1312,14 +1310,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x2f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -1446,14 +1444,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mfma_f32_32x32x2f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -1844,127 +1842,127 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s23
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s25
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s26
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s27
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s28
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s28
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s29
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s30
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s31
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s29
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s30
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s31
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a16, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a17, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s3
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s5
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s6
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s7
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s8
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a22, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s8
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a23, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s9
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s10
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s9
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[38:39], 0x0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s19
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s11
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s21
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s14
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s15
-; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s19
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s13
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
+; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s3
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
+; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[5:6], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37]
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
@@ -2106,127 +2104,127 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s23
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s25
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v0
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s26
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s27
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s28
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s28
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s29
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s30
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s31
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s29
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s30
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s31
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v0
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a16, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a17, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s3
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s5
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v0
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s6
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s7
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s8
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a22, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s8
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a23, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s9
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s10
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s9
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[38:39], 0x0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s19
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s11
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s21
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s14
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s15
-; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s19
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s13
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
+; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s3
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
+; LIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[5:6], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37]
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16
; LIT-SRCC-NEXT: s_endpgm
@@ -2249,8 +2247,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: v_mov_b32_e32 v2, s18
; GFX90A-NEXT: v_mov_b32_e32 v3, s19
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v34, s0
-; GFX90A-NEXT: v_mov_b32_e32 v35, s1
+; GFX90A-NEXT: v_pk_mov_b32 v[34:35], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: v_mov_b32_e32 v5, s21
; GFX90A-NEXT: v_mov_b32_e32 v6, s22
@@ -2275,8 +2272,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: v_mov_b32_e32 v29, s13
; GFX90A-NEXT: v_mov_b32_e32 v30, s14
; GFX90A-NEXT: v_mov_b32_e32 v31, s15
-; GFX90A-NEXT: v_mov_b32_e32 v36, s2
-; GFX90A-NEXT: v_mov_b32_e32 v37, s3
+; GFX90A-NEXT: v_pk_mov_b32 v[36:37], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
; GFX90A-NEXT: v_mfma_f32_32x32x4f16 v[0:31], v[34:35], v[36:37], v[0:31] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT: s_nop 15
@@ -2350,8 +2346,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: v_mov_b32_e32 v2, s18
; GFX942-NEXT: v_mov_b32_e32 v3, s19
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v34, s0
-; GFX942-NEXT: v_mov_b32_e32 v35, s1
+; GFX942-NEXT: v_mov_b64_e32 v[34:35], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v4, s20
; GFX942-NEXT: v_mov_b32_e32 v5, s21
; GFX942-NEXT: v_mov_b32_e32 v6, s22
@@ -2376,8 +2371,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: v_mov_b32_e32 v29, s13
; GFX942-NEXT: v_mov_b32_e32 v30, s14
; GFX942-NEXT: v_mov_b32_e32 v31, s15
-; GFX942-NEXT: v_mov_b32_e32 v36, s2
-; GFX942-NEXT: v_mov_b32_e32 v37, s3
+; GFX942-NEXT: v_mov_b64_e32 v[36:37], s[2:3]
; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[34:35], v[36:37], v[0:31] cbsz:1 abid:2 blgp:3
; GFX942-NEXT: s_nop 15
@@ -2451,8 +2445,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18
; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, s0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v35, s1
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[0:1]
; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21
; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22
@@ -2477,8 +2470,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13
; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14
; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v36, s2
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v37, s3
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[2:3]
; GFX942-VGPR-NEXT: s_nop 1
; GFX942-VGPR-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[34:35], v[36:37], v[0:31] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT: s_nop 15
@@ -2547,71 +2539,70 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-LABEL: test_mfma_f32_16x16x4f16:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v12, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s20
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s2
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s4
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s6
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s8
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s9
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s10
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s11
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s12
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s4
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s5
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s7
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s8
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s10
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s11
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s21
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s14
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v13, s15
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v13
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s13
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s14
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v3
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s22
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x4f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_16x16x4f16:
@@ -2686,71 +2677,70 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-LABEL: test_mfma_f32_16x16x4f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v12, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s20
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s2
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s4
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s6
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s8
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s9
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s10
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s11
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s12
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s4
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s5
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s7
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s8
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s10
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s11
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s21
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s14
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, s15
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v4
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v13
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s13
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s14
+; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v3
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s22
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s23
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: v_mfma_f32_16x16x4f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_16x16x4f16:
@@ -2760,10 +2750,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v16, s20
-; GFX90A-NEXT: v_mov_b32_e32 v17, s21
-; GFX90A-NEXT: v_mov_b32_e32 v18, s22
-; GFX90A-NEXT: v_mov_b32_e32 v19, s23
+; GFX90A-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
@@ -2816,10 +2804,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v16, s20
-; GFX942-NEXT: v_mov_b32_e32 v17, s21
-; GFX942-NEXT: v_mov_b32_e32 v18, s22
-; GFX942-NEXT: v_mov_b32_e32 v19, s23
+; GFX942-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
@@ -2872,10 +2858,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s20
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s21
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s22
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s23
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
@@ -3032,10 +3016,8 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v6, s4
-; GFX90A-NEXT: v_mov_b32_e32 v7, s5
-; GFX90A-NEXT: v_mov_b32_e32 v8, s6
-; GFX90A-NEXT: v_mov_b32_e32 v9, s7
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
@@ -3070,10 +3052,8 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX942-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, s4
-; GFX942-NEXT: v_mov_b32_e32 v7, s5
-; GFX942-NEXT: v_mov_b32_e32 v8, s6
-; GFX942-NEXT: v_mov_b32_e32 v9, s7
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GFX942-NEXT: s_nop 1
@@ -3108,10 +3088,8 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-VGPR-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s4
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s5
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s6
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s7
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GFX942-VGPR-NEXT: s_nop 1
@@ -3158,37 +3136,37 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s20
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s2
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s4
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s5
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s4
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s5
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s8
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s7
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s8
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s10
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s11
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s10
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s11
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s21
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s14
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s13
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s14
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v3
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s22
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s23
@@ -3196,14 +3174,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -3298,37 +3276,37 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s20
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s2
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v17
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s4
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s5
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s4
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s5
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s6
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v17
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s8
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s7
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s8
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s9
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v17
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s10
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s11
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s10
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s11
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s12
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s21
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v17
-; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s14
+; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s13
+; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s14
; LIT-SRCC-NEXT: v_mov_b32_e32 v17, s15
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v3
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v17
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s22
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s23
@@ -3336,14 +3314,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -3365,10 +3343,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v16, s20
-; GFX90A-NEXT: v_mov_b32_e32 v17, s21
-; GFX90A-NEXT: v_mov_b32_e32 v18, s22
-; GFX90A-NEXT: v_mov_b32_e32 v19, s23
+; GFX90A-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
@@ -3423,10 +3399,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v16, s20
-; GFX942-NEXT: v_mov_b32_e32 v17, s21
-; GFX942-NEXT: v_mov_b32_e32 v18, s22
-; GFX942-NEXT: v_mov_b32_e32 v19, s23
+; GFX942-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
@@ -3479,10 +3453,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s20
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s21
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s22
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s23
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
@@ -3639,10 +3611,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v6, s4
-; GFX90A-NEXT: v_mov_b32_e32 v7, s5
-; GFX90A-NEXT: v_mov_b32_e32 v8, s6
-; GFX90A-NEXT: v_mov_b32_e32 v9, s7
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
@@ -3677,10 +3647,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX942-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, s4
-; GFX942-NEXT: v_mov_b32_e32 v7, s5
-; GFX942-NEXT: v_mov_b32_e32 v8, s6
-; GFX942-NEXT: v_mov_b32_e32 v9, s7
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GFX942-NEXT: s_nop 1
@@ -3715,10 +3683,8 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-VGPR-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s4
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s5
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s6
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s7
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GFX942-VGPR-NEXT: s_nop 1
@@ -3758,7 +3724,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-LABEL: test_mfma_i32_32x32x4i8:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -3815,7 +3781,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -3824,7 +3790,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -3833,47 +3799,61 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
+; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35]
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_i32_32x32x4i8:
@@ -3999,7 +3979,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-LABEL: test_mfma_i32_32x32x4i8:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -4056,7 +4036,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -4065,7 +4045,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, 1
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -4074,47 +4054,61 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
+; LIT-SRCC-NEXT: s_nop 0
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35]
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_i32_32x32x4i8:
@@ -4456,27 +4450,26 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_i32_16x16x4i8:
@@ -4589,27 +4582,26 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_i32_16x16x4i8:
@@ -4796,30 +4788,37 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 64
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
@@ -4877,27 +4876,27 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; LIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
@@ -5748,59 +5747,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
@@ -6005,59 +6004,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
@@ -6409,14 +6408,14 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -6543,14 +6542,14 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
@@ -7148,32 +7147,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
@@ -7230,33 +7229,30 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v8, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, 1.0
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:32
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
@@ -7491,32 +7487,32 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
@@ -7582,33 +7578,31 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, 0x40004000
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, v2
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v13, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
; LIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], 1.0
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a7
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a6
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a8
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a4
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[0:3], s[0:1] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[4:7], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[8:11], s[0:1] offset:16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[9:12], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
@@ -7884,60 +7878,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
@@ -8031,55 +8025,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v14, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v24, 0
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, 0
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a24
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a27
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a8
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:112
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[4:7], s[0:1] offset:96
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[8:11], s[0:1] offset:80
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a18
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:48
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[10:13], s[0:1] offset:64
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1] offset:112
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:96
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[8:11], s[0:1] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[12:15], s[0:1] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[16:19], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[20:23], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
@@ -8460,32 +8450,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; NOLIT-SRCC-NEXT: s_nop 9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
@@ -8561,32 +8551,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; LIT-SRCC-NEXT: s_nop 9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_nop 0
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
@@ -8845,60 +8835,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
@@ -9028,60 +9018,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_nop 0
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
@@ -9936,22 +9926,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
@@ -9966,10 +9956,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
@@ -10126,22 +10116,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
@@ -10156,10 +10146,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index aeeb5958e9ca3..cc7d809e4953a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -2513,36 +2513,25 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v5, s12
-; SDAG-NEXT: v_mov_b32_e32 v6, s13
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
+; SDAG-NEXT: v_mov_b32_e32 v5, s28
+; SDAG-NEXT: v_mov_b32_e32 v6, s29
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[30:31]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2572,36 +2561,25 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s13
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s28
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s29
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[30:31]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2640,27 +2618,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; SDAG-NEXT: v_mov_b32_e32 v5, -2
; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -2698,27 +2668,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: s_nop 0
; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -2761,27 +2723,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -2819,27 +2773,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 1.0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: s_nop 0
; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -2882,27 +2828,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; SDAG-NEXT: v_mov_b32_e32 v5, -2
; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -2940,27 +2878,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: s_nop 0
; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -3003,27 +2933,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
@@ -3061,27 +2983,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: s_nop 0
; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..a8f7b8b597b59 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -7306,23 +7306,15 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7384,23 +7376,15 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[16:17]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7469,22 +7453,14 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-NEXT: v_mov_b32_e32 v17, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7547,22 +7523,14 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 0x41
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[8:9]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[16:17]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7624,27 +7592,18 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[36:37]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[38:39]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[40:41]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[42:43]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[44:45]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[46:47]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[48:49]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[50:51]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -7655,36 +7614,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b32_e32 v16, s0
; SDAG-NEXT: v_mov_b32_e32 v17, s1
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel_hi:[0,0,0]
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_nop 1
; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
@@ -7754,27 +7705,18 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s23
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[36:37]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[38:39]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[40:41]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[42:43]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[44:45]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[46:47]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[48:49]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[50:51]
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s10
@@ -7793,36 +7735,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[10:11]
; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], v0, v1 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s17
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s13
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[8:9]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[4:5], v[16:19], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[6:7], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
@@ -7922,28 +7856,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v16, 42
; SDAG-NEXT: v_mov_b32_e32 v17, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[36:37]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[38:39]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[40:41]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[42:43]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[44:45]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[46:47]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[48:49]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[50:51]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -7952,36 +7877,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_nop 1
; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
@@ -8049,28 +7966,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, 42
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s23
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s27
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[36:37]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[38:39]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[40:41]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[42:43]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[44:45]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[46:47]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[48:49]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[50:51]
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s10
@@ -8087,36 +7995,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a15, s23
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[10:11]
; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], v1, v0 op_sel_hi:[0,0,0] blgp:2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s17
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s13
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[8:9]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[4:5], v[16:19], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[6:7], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
@@ -8214,26 +8114,17 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s12
-; SDAG-NEXT: v_mov_b32_e32 v33, s13
-; SDAG-NEXT: v_mov_b32_e32 v34, s14
-; SDAG-NEXT: v_mov_b32_e32 v35, s15
-; SDAG-NEXT: v_mov_b32_e32 v36, s16
-; SDAG-NEXT: v_mov_b32_e32 v37, s17
-; SDAG-NEXT: v_mov_b32_e32 v38, s18
-; SDAG-NEXT: v_mov_b32_e32 v39, s19
-; SDAG-NEXT: v_mov_b32_e32 v40, s20
-; SDAG-NEXT: v_mov_b32_e32 v41, s21
-; SDAG-NEXT: v_mov_b32_e32 v42, s22
-; SDAG-NEXT: v_mov_b32_e32 v43, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v44, s24
-; SDAG-NEXT: v_mov_b32_e32 v45, s25
-; SDAG-NEXT: v_mov_b32_e32 v46, s26
-; SDAG-NEXT: v_mov_b32_e32 v47, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[38:39]
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[40:41]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[42:43]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[44:45]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[46:47]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[48:49]
+; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[50:51]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -8244,34 +8135,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; SDAG-NEXT: s_nop 14
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; SDAG-NEXT: s_nop 10
; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -8331,26 +8214,17 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v34, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v35, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v38, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v39, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v40, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v41, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v42, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v43, s23
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v44, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v45, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v46, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v47, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], s[38:39]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[36:37], s[40:41]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[42:43]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[40:41], s[44:45]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[42:43], s[46:47]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[44:45], s[48:49]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[46:47], s[50:51]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -8361,34 +8235,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; AGPR-SDAG-NEXT: s_nop 14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; AGPR-SDAG-NEXT: s_nop 10
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -8454,28 +8320,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v16, 42
; SDAG-NEXT: v_mov_b32_e32 v17, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[36:37]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[38:39]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[40:41]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[42:43]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[44:45]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[46:47]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[48:49]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[50:51]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -8484,36 +8341,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_nop 1
; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
@@ -8581,28 +8430,19 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 42
; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s23
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s27
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[36:37]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[38:39]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[40:41]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], s[42:43]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[44:45]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[46:47]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[48:49]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[50:51]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -8611,36 +8451,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-SDAG-NEXT: s_nop 1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], s[14:15]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[10:11]
; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[12:13]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[36:37], s[8:9]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[28:31], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[32:35], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[36:39], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
+; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
index e0576b8054b2b..59120a8543cd9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
@@ -18,18 +18,17 @@ define amdgpu_kernel void @test_p0(ptr addrspace(1) %out, ptr %src0) {
define amdgpu_kernel void @test_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0) {
; GFX11-SDAG-LABEL: test_v3p0:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x2
-; GFX11-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x54
-; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x44
-; GFX11-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, s7
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, s13
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v1, s9
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s10
; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: global_store_b64 v6, v[4:5], s[4:5] offset:16
-; GFX11-SDAG-NEXT: global_store_b128 v6, v[0:3], s[4:5]
+; GFX11-SDAG-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX11-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
; GFX11-SDAG-NEXT: s_endpgm
%v = call <3 x ptr> @llvm.amdgcn.permlane64.v3p0(<3 x ptr> %src0)
store <3 x ptr> %v, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
index 38cee652160ed..70d5b14e0558b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
@@ -266,11 +266,11 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-ILP: ; %bb.0: ; %entry
; GCN-ILP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-ILP-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-ILP-NEXT: v_and_b32_e32 v32, 0x1ff80, v0
-; GCN-ILP-NEXT: v_mov_b32_e32 v33, 1.0
-; GCN-ILP-NEXT: v_mov_b32_e32 v34, 2.0
+; GCN-ILP-NEXT: v_and_b32_e32 v34, 0x1ff80, v0
+; GCN-ILP-NEXT: v_mov_b32_e32 v32, 1.0
+; GCN-ILP-NEXT: v_mov_b32_e32 v33, 2.0
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_add_u32_e32 v35, s0, v32
+; GCN-ILP-NEXT: v_add_u32_e32 v35, s0, v34
; GCN-ILP-NEXT: ds_read_b128 v[12:15], v35 offset:48
; GCN-ILP-NEXT: ds_read_b128 v[8:11], v35 offset:32
; GCN-ILP-NEXT: ds_read_b128 v[4:7], v35 offset:16
@@ -280,20 +280,20 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-ILP-NEXT: ds_read_b128 v[24:27], v35 offset:96
; GCN-ILP-NEXT: ds_read_b128 v[28:31], v35 offset:112
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31]
-; GCN-ILP-NEXT: v_add_u32_e32 v32, s1, v32
+; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GCN-ILP-NEXT: v_add_u32_e32 v34, s1, v34
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: s_nop 15
; GCN-ILP-NEXT: s_nop 1
-; GCN-ILP-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GCN-ILP-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GCN-ILP-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GCN-ILP-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GCN-ILP-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GCN-ILP-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GCN-ILP-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GCN-ILP-NEXT: ds_write_b128 v32, v[0:3]
+; GCN-ILP-NEXT: ds_write_b128 v34, v[28:31] offset:112
+; GCN-ILP-NEXT: ds_write_b128 v34, v[24:27] offset:96
+; GCN-ILP-NEXT: ds_write_b128 v34, v[20:23] offset:80
+; GCN-ILP-NEXT: ds_write_b128 v34, v[16:19] offset:64
+; GCN-ILP-NEXT: ds_write_b128 v34, v[12:15] offset:48
+; GCN-ILP-NEXT: ds_write_b128 v34, v[8:11] offset:32
+; GCN-ILP-NEXT: ds_write_b128 v34, v[4:7] offset:16
+; GCN-ILP-NEXT: ds_write_b128 v34, v[0:3]
; GCN-ILP-NEXT: ds_read_b128 v[0:3], v35 offset:8192
; GCN-ILP-NEXT: ds_read_b128 v[4:7], v35 offset:8208
; GCN-ILP-NEXT: ds_read_b128 v[8:11], v35 offset:8224
@@ -303,18 +303,18 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-ILP-NEXT: ds_read_b128 v[24:27], v35 offset:8288
; GCN-ILP-NEXT: ds_read_b128 v[28:31], v35 offset:8304
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31]
-; GCN-ILP-NEXT: v_mov_b32_e32 v32, s1
+; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GCN-ILP-NEXT: v_mov_b32_e32 v34, s1
; GCN-ILP-NEXT: s_nop 15
; GCN-ILP-NEXT: s_nop 1
-; GCN-ILP-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GCN-ILP-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GCN-ILP-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GCN-ILP-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GCN-ILP-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GCN-ILP-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GCN-ILP-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GCN-ILP-NEXT: ds_write_b128 v32, v[4:7] offset:8208
+; GCN-ILP-NEXT: ds_write_b128 v34, v[24:27] offset:8288
+; GCN-ILP-NEXT: ds_write_b128 v34, v[28:31] offset:8304
+; GCN-ILP-NEXT: ds_write_b128 v34, v[16:19] offset:8256
+; GCN-ILP-NEXT: ds_write_b128 v34, v[20:23] offset:8272
+; GCN-ILP-NEXT: ds_write_b128 v34, v[8:11] offset:8224
+; GCN-ILP-NEXT: ds_write_b128 v34, v[12:15] offset:8240
+; GCN-ILP-NEXT: ds_write_b128 v34, v[0:3] offset:8192
+; GCN-ILP-NEXT: ds_write_b128 v34, v[4:7] offset:8208
; GCN-ILP-NEXT: ds_read_b128 v[0:3], v35 offset:24576
; GCN-ILP-NEXT: ds_read_b128 v[4:7], v35 offset:24592
; GCN-ILP-NEXT: ds_read_b128 v[8:11], v35 offset:24608
@@ -324,62 +324,62 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-ILP-NEXT: ds_read_b128 v[24:27], v35 offset:24672
; GCN-ILP-NEXT: ds_read_b128 v[28:31], v35 offset:24688
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31]
+; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
; GCN-ILP-NEXT: s_nop 15
; GCN-ILP-NEXT: s_nop 2
-; GCN-ILP-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GCN-ILP-NEXT: ds_write_b128 v34, v[4:7] offset:16400
; GCN-ILP-NEXT: ds_read_b128 v[4:7], v35 offset:49168
-; GCN-ILP-NEXT: ds_write_b128 v32, v[0:3] offset:16384
+; GCN-ILP-NEXT: ds_write_b128 v34, v[0:3] offset:16384
; GCN-ILP-NEXT: ds_read_b128 v[0:3], v35 offset:49152
-; GCN-ILP-NEXT: ds_write_b128 v32, v[12:15] offset:16432
+; GCN-ILP-NEXT: ds_write_b128 v34, v[12:15] offset:16432
; GCN-ILP-NEXT: ds_read_b128 v[12:15], v35 offset:49200
-; GCN-ILP-NEXT: ds_write_b128 v32, v[8:11] offset:16416
+; GCN-ILP-NEXT: ds_write_b128 v34, v[8:11] offset:16416
; GCN-ILP-NEXT: ds_read_b128 v[8:11], v35 offset:49184
-; GCN-ILP-NEXT: ds_write_b128 v32, v[20:23] offset:16464
+; GCN-ILP-NEXT: ds_write_b128 v34, v[20:23] offset:16464
; GCN-ILP-NEXT: ds_read_b128 v[20:23], v35 offset:49232
-; GCN-ILP-NEXT: ds_write_b128 v32, v[16:19] offset:16448
+; GCN-ILP-NEXT: ds_write_b128 v34, v[16:19] offset:16448
; GCN-ILP-NEXT: ds_read_b128 v[16:19], v35 offset:49216
-; GCN-ILP-NEXT: ds_write_b128 v32, v[28:31] offset:16496
+; GCN-ILP-NEXT: ds_write_b128 v34, v[28:31] offset:16496
; GCN-ILP-NEXT: ds_read_b128 v[28:31], v35 offset:49264
-; GCN-ILP-NEXT: ds_write_b128 v32, v[24:27] offset:16480
+; GCN-ILP-NEXT: ds_write_b128 v34, v[24:27] offset:16480
; GCN-ILP-NEXT: ds_read_b128 v[24:27], v35 offset:49248
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31]
+; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
; GCN-ILP-NEXT: v_add_u32_e32 v35, 0x6000, v35
; GCN-ILP-NEXT: s_nop 15
; GCN-ILP-NEXT: s_nop 1
-; GCN-ILP-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GCN-ILP-NEXT: ds_write_b128 v34, v[4:7] offset:24592
; GCN-ILP-NEXT: ds_read_b128 v[4:7], v35 offset:57360
-; GCN-ILP-NEXT: ds_write_b128 v32, v[0:3] offset:24576
+; GCN-ILP-NEXT: ds_write_b128 v34, v[0:3] offset:24576
; GCN-ILP-NEXT: ds_read_b128 v[0:3], v35 offset:57344
-; GCN-ILP-NEXT: ds_write_b128 v32, v[12:15] offset:24624
+; GCN-ILP-NEXT: ds_write_b128 v34, v[12:15] offset:24624
; GCN-ILP-NEXT: ds_read_b128 v[12:15], v35 offset:57392
-; GCN-ILP-NEXT: ds_write_b128 v32, v[8:11] offset:24608
+; GCN-ILP-NEXT: ds_write_b128 v34, v[8:11] offset:24608
; GCN-ILP-NEXT: ds_read_b128 v[8:11], v35 offset:57376
-; GCN-ILP-NEXT: ds_write_b128 v32, v[20:23] offset:24656
+; GCN-ILP-NEXT: ds_write_b128 v34, v[20:23] offset:24656
; GCN-ILP-NEXT: ds_read_b128 v[20:23], v35 offset:57424
-; GCN-ILP-NEXT: ds_write_b128 v32, v[16:19] offset:24640
+; GCN-ILP-NEXT: ds_write_b128 v34, v[16:19] offset:24640
; GCN-ILP-NEXT: ds_read_b128 v[16:19], v35 offset:57408
-; GCN-ILP-NEXT: ds_write_b128 v32, v[28:31] offset:24688
+; GCN-ILP-NEXT: ds_write_b128 v34, v[28:31] offset:24688
; GCN-ILP-NEXT: ds_read_b128 v[28:31], v35 offset:57456
-; GCN-ILP-NEXT: ds_write_b128 v32, v[24:27] offset:24672
+; GCN-ILP-NEXT: ds_write_b128 v34, v[24:27] offset:24672
; GCN-ILP-NEXT: ds_read_b128 v[24:27], v35 offset:57440
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31]
+; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: s_nop 15
; GCN-ILP-NEXT: s_nop 2
-; GCN-ILP-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GCN-ILP-NEXT: ds_write_b128 v32, v[28:31] offset:32880
-; GCN-ILP-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GCN-ILP-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GCN-ILP-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GCN-ILP-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GCN-ILP-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GCN-ILP-NEXT: ds_write_b128 v32, v[4:7] offset:32784
+; GCN-ILP-NEXT: ds_write_b128 v34, v[24:27] offset:32864
+; GCN-ILP-NEXT: ds_write_b128 v34, v[28:31] offset:32880
+; GCN-ILP-NEXT: ds_write_b128 v34, v[16:19] offset:32832
+; GCN-ILP-NEXT: ds_write_b128 v34, v[20:23] offset:32848
+; GCN-ILP-NEXT: ds_write_b128 v34, v[8:11] offset:32800
+; GCN-ILP-NEXT: ds_write_b128 v34, v[12:15] offset:32816
+; GCN-ILP-NEXT: ds_write_b128 v34, v[0:3] offset:32768
+; GCN-ILP-NEXT: ds_write_b128 v34, v[4:7] offset:32784
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 96edd7876e981..07c82bdf12ede 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -863,23 +863,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; SDAG-NEXT: v_mov_b32_e32 v1, 2.0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v3, s0, v0
-; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:112
-; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:96
-; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:80
-; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:64
-; SDAG-NEXT: ds_read_b128 a[32:35], v3
-; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:16
-; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:32
-; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:48
+; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v3
+; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:48
; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:8304
-; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:8288
-; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:8272
-; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:8256
-; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:8240
-; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:8224
-; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:8208
-; SDAG-NEXT: ds_read_b128 a[0:3], v3 offset:8192
+; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; SDAG-NEXT: ds_read_b128 a[32:35], v3 offset:8192
; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:24688
; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:24672
; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:24656
@@ -905,33 +905,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; SDAG-NEXT: ds_read_b128 a[136:139], v4 offset:57376
; SDAG-NEXT: ds_read_b128 a[140:143], v4 offset:57392
; SDAG-NEXT: s_waitcnt lgkmcnt(14)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
; SDAG-NEXT: s_waitcnt lgkmcnt(8)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
; SDAG-NEXT: v_mov_b32_e32 v0, s1
-; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
@@ -1236,23 +1236,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, 2.0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_add_u32_e32 v3, s0, v0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:112
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:96
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:80
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:64
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:16
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:32
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:48
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:48
; EXACTCUTOFF-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:8304
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3 offset:8192
+; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3 offset:8192
; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v3 offset:24688
; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v3 offset:24672
; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v3 offset:24656
@@ -1278,33 +1278,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v4 offset:57376
; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v4 offset:57392
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(14)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s1, v0
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(8)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
; EXACTCUTOFF-NEXT: s_nop 11
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35]
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3]
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448
@@ -1994,14 +1994,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: v_cvt_i32_f32_e32 v5, v5
; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
; SDAG-NEXT: v_add_u32_e32 v1, s6, v0
-; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112
-; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96
-; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80
-; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64
-; SDAG-NEXT: ds_read_b128 a[32:35], v1
-; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16
-; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32
-; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v1
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:48
; SDAG-NEXT: v_mov_b32_e32 v9, 1.0
; SDAG-NEXT: v_ldexp_f32 v4, v4, v5
; SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
@@ -2016,19 +2016,19 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: v_fma_f32 v10, s1, v3, -v10
; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
; SDAG-NEXT: v_fmac_f32_e32 v10, s1, v7
-; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:8304
; SDAG-NEXT: s_waitcnt lgkmcnt(1)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
; SDAG-NEXT: v_add_f32_e32 v4, v12, v10
; SDAG-NEXT: v_exp_f32_e32 v4, v4
; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11
-; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:8192
; SDAG-NEXT: v_ldexp_f32 v4, v4, v10
; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
@@ -2037,7 +2037,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: v_mul_f32_e32 v10, s2, v3
; SDAG-NEXT: v_rndne_f32_e32 v11, v10
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
; SDAG-NEXT: v_fma_f32 v4, s2, v3, -v10
; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11
; SDAG-NEXT: v_fmac_f32_e32 v4, s2, v7
@@ -2105,16 +2105,16 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
; SDAG-NEXT: v_add_u32_e32 v0, s7, v0
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
; SDAG-NEXT: s_waitcnt lgkmcnt(1)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
; SDAG-NEXT: v_mov_b32_e32 v0, s7
; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5
; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
@@ -2127,14 +2127,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
@@ -2544,14 +2544,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v5, v5
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s6, v0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:112
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:96
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:80
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:64
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:16
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:32
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:48
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:48
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v9, 1.0
; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v5
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
@@ -2566,19 +2566,19 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: v_fma_f32 v10, s1, v3, -v10
; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v10, s1, v7
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:8304
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v10
; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1 offset:8192
; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v10
; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
@@ -2587,7 +2587,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s2, v3
; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
; EXACTCUTOFF-NEXT: v_fma_f32 v4, s2, v3, -v10
; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11
; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s2, v7
@@ -2655,16 +2655,16 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35]
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3]
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s7
; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
@@ -2677,14 +2677,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
index 0f8c5ca34c501..0721d740c9dec 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
@@ -1711,10 +1711,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v2, s12
; SDAG-NEXT: v_mov_b32_e32 v3, s13
; SDAG-NEXT: v_mov_b32_e32 v4, s14
@@ -1767,10 +1765,8 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
@@ -2034,10 +2030,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v18, s12
; SDAG-NEXT: v_mov_b32_e32 v19, s13
; SDAG-NEXT: v_mov_b32_e32 v20, s14
@@ -2102,10 +2096,8 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
@@ -2580,10 +2572,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v2, s12
; SDAG-NEXT: v_mov_b32_e32 v3, s13
; SDAG-NEXT: v_mov_b32_e32 v4, s14
@@ -2636,10 +2626,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
@@ -2901,10 +2889,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v2, s12
; SDAG-NEXT: v_mov_b32_e32 v3, s13
; SDAG-NEXT: v_mov_b32_e32 v4, s14
@@ -2957,10 +2943,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
@@ -3222,10 +3206,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v2, s12
; SDAG-NEXT: v_mov_b32_e32 v3, s13
; SDAG-NEXT: v_mov_b32_e32 v4, s14
@@ -3278,10 +3260,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
@@ -3543,10 +3523,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v2, s12
; SDAG-NEXT: v_mov_b32_e32 v3, s13
; SDAG-NEXT: v_mov_b32_e32 v4, s14
@@ -3599,10 +3577,8 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
@@ -3866,10 +3842,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v18, s12
; SDAG-NEXT: v_mov_b32_e32 v19, s13
; SDAG-NEXT: v_mov_b32_e32 v20, s14
@@ -3934,10 +3908,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
@@ -4414,10 +4386,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v18, s12
; SDAG-NEXT: v_mov_b32_e32 v19, s13
; SDAG-NEXT: v_mov_b32_e32 v20, s14
@@ -4482,10 +4452,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
@@ -4962,10 +4930,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v18, s12
; SDAG-NEXT: v_mov_b32_e32 v19, s13
; SDAG-NEXT: v_mov_b32_e32 v20, s14
@@ -5030,10 +4996,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
@@ -5510,10 +5474,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v18, s12
; SDAG-NEXT: v_mov_b32_e32 v19, s13
; SDAG-NEXT: v_mov_b32_e32 v20, s14
@@ -5578,10 +5540,8 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[8:9]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 1536ac5112e7b..bbfcc2975a14c 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -447,10 +447,10 @@ define amdgpu_kernel void @constant_load_v16i16(ptr addrspace(1) %out, ptr addrs
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s5
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s6
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s7
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v4, s0
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, s1
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s2
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, s3
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s2
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, s1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v4, s0
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0
; GCN-NOHSA-SI-NEXT: s_endpgm
@@ -465,21 +465,21 @@ define amdgpu_kernel void @constant_load_v16i16(ptr addrspace(1) %out, ptr addrs
; GCN-HSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GCN-HSA-NEXT: s_add_u32 s10, s8, 16
; GCN-HSA-NEXT: s_addc_u32 s11, s9, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v6, s10
-; GCN-HSA-NEXT: v_mov_b32_e32 v7, s11
+; GCN-HSA-NEXT: v_mov_b32_e32 v10, s10
+; GCN-HSA-NEXT: v_mov_b32_e32 v11, s11
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s4
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s6
; GCN-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GCN-HSA-NEXT: v_mov_b32_e32 v4, s0
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GCN-HSA-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GCN-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, s8
+; GCN-HSA-NEXT: v_mov_b32_e32 v9, s9
; GCN-HSA-NEXT: v_mov_b32_e32 v7, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v0, s8
-; GCN-HSA-NEXT: v_mov_b32_e32 v1, s9
-; GCN-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GCN-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCN-HSA-NEXT: s_endpgm
;
; GCN-NOHSA-VI-LABEL: constant_load_v16i16:
@@ -489,21 +489,21 @@ define amdgpu_kernel void @constant_load_v16i16(ptr addrspace(1) %out, ptr addrs
; GCN-NOHSA-VI-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GCN-NOHSA-VI-NEXT: s_add_u32 s10, s8, 16
; GCN-NOHSA-VI-NEXT: s_addc_u32 s11, s9, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v6, s10
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v7, s11
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v10, s10
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v11, s11
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s4
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s5
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s6
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s7
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s1
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v6, s2
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v8, s8
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v9, s9
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v7, s3
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s8
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s9
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v6, s2
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s1
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCN-NOHSA-VI-NEXT: s_endpgm
;
; EG-LABEL: constant_load_v16i16:
@@ -533,9 +533,9 @@ define amdgpu_kernel void @constant_load_v16i16(ptr addrspace(1) %out, ptr addrs
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s1
-; GFX12-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v7, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, s2
+; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v7, s3
+; GFX12-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
+; GFX12-NEXT: v_mov_b32_e32 v4, s0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v8, v[0:3], s[8:9] offset:16
; GFX12-NEXT: global_store_b128 v8, v[4:7], s[8:9]
@@ -7783,137 +7783,136 @@ define amdgpu_kernel void @constant_zextload_v32i16_to_v32i64(ptr addrspace(1) %
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s20, s15, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s21, s14, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s22, s13, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s23, s12, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s24, s11, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s25, s10, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s26, s9, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s27, s8, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s28, s7, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s29, s6, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s30, s5, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s31, s4, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s33, s3, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s34, s2, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s19, s1, 16
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s18, s0, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s0, s0, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s19, s1, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s1, s1, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s20, s2, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s2, s2, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s21, s3, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s3, s3, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s22, s4, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s23, s4, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s24, s5, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s25, s5, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s26, s6, 16
+; GCN-NOHSA-VI-NEXT: s_and_b32 s35, s2, 0xffff
+; GCN-NOHSA-VI-NEXT: s_and_b32 s36, s3, 0xffff
+; GCN-NOHSA-VI-NEXT: s_and_b32 s4, s4, 0xffff
+; GCN-NOHSA-VI-NEXT: s_and_b32 s5, s5, 0xffff
; GCN-NOHSA-VI-NEXT: s_and_b32 s6, s6, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s27, s7, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s7, s7, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s28, s8, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s8, s8, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s29, s9, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s9, s9, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s30, s10, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s10, s10, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s31, s11, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s11, s11, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s33, s12, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s12, s12, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s34, s13, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s13, s13, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s35, s14, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s14, s14, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s4, s15, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s5, s15, 0xffff
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xf0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s5
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xe0
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: s_and_b32 s2, s15, 0xffff
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xf0
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xe0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s20
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xd0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xd0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s14
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s35
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s21
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xc0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xc0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s13
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s34
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s22
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xb0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xb0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s12
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s33
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s23
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0xa0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0xa0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s11
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s31
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s24
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0x90
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0x90
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s10
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s30
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s25
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0x80
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0x80
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s9
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s29
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s26
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0x70
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0x70
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s8
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s28
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s27
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0x60
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0x60
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s7
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s27
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s28
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 0x50
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 0x50
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s6
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s26
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 64
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s25
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s24
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s29
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 48
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s23
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s22
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 64
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s30
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s21
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 48
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s31
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 32
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s36
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s33
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 16
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s20
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s35
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s34
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
@@ -8558,211 +8557,208 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s22, s0, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s26, s1
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s28, s1, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s30, s2, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s34, s3
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s36, s3, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s40, s4, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s42, s5
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s44, s5, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s46, s6, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s48, s7
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s50, s7, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s52, s8, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s54, s9
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s56, s9, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s58, s10, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s40, s15, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s42, s15
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s46, s14, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s48, s13, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s54, s13
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s56, s12, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s58, s11, 16
; GCN-NOHSA-VI-NEXT: s_mov_b32 s60, s11
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s62, s11, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s66, s12, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s68, s13
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s70, s13, 16
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s74, s14, 16
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s76, s15
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s78, s15, 16
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[18:19], s[0:1], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[20:21], s[6:7], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[24:25], s[8:9], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[38:39], s[10:11], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[64:65], s[12:13], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[72:73], s[14:15], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[4:5], s[22:23], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[8:9], s[26:27], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[6:7], s[28:29], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[10:11], s[30:31], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[14:15], s[34:35], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[12:13], s[36:37], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[22:23], s[40:41], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[28:29], s[42:43], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[26:27], s[44:45], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[30:31], s[46:47], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[36:37], s[48:49], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[34:35], s[50:51], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[40:41], s[52:53], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[44:45], s[54:55], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[42:43], s[56:57], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[46:47], s[58:59], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[48:49], s[60:61], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[50:51], s[62:63], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[52:53], s[66:67], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[54:55], s[68:69], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[56:57], s[70:71], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[58:59], s[74:75], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[60:61], s[76:77], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[62:63], s[78:79], 0x100000
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s60
-; GCN-NOHSA-VI-NEXT: s_add_u32 s60, s16, 0xf0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s61
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s61, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s62
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s63
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s60
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s61
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s62, s10, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s50, s9, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s52, s9
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s38, s8, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s34, s7, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s36, s7
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s28, s6, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s24, s5, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s26, s5
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s64, s4, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s66, s3, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s68, s3
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s70, s2, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s72, s1, 16
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s74, s1
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s76, s0, 16
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[18:19], s[2:3], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[20:21], s[4:5], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[22:23], s[6:7], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[30:31], s[8:9], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[44:45], s[10:11], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[78:79], s[12:13], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[80:81], s[14:15], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[2:3], s[76:77], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[6:7], s[74:75], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[4:5], s[72:73], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[8:9], s[70:71], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[12:13], s[68:69], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[10:11], s[66:67], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[14:15], s[64:65], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x100000
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s40
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xf0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s41
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xe0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s42
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s43
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s58
-; GCN-NOHSA-VI-NEXT: s_add_u32 s58, s16, 0xe0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s59
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s59, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s72
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s73
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s58
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s59
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xd0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s80
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s81
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s46
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s47
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xc0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s54
-; GCN-NOHSA-VI-NEXT: s_add_u32 s54, s16, 0xd0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s55
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s55, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s48
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s49
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xb0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s78
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s79
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s56
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s57
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s54
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s55
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s52
-; GCN-NOHSA-VI-NEXT: s_add_u32 s52, s16, 0xc0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s53
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s53, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s64
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s65
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s52
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s53
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s48
-; GCN-NOHSA-VI-NEXT: s_add_u32 s48, s16, 0xb0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s49
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s49, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s50
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s51
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s48
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s49
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s38
-; GCN-NOHSA-VI-NEXT: s_add_u32 s38, s16, 0xa0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s39
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s39, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s38
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s39
-; GCN-NOHSA-VI-NEXT: s_add_u32 s38, s16, 0x90
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s46
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s47
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s39, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0xa0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s60
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s61
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s58
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s59
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s38
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT: s_add_u32 s40, s16, 0x90
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s44
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s45
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s42
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s43
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s39
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s62
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s63
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s41, s17, 0
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s52
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s53
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s50
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s51
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s41
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s24
-; GCN-NOHSA-VI-NEXT: s_add_u32 s24, s16, 0x80
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s25
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s25, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s24
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s25
-; GCN-NOHSA-VI-NEXT: s_add_u32 s24, s16, 0x70
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s40
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s41
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s25, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s30
+; GCN-NOHSA-VI-NEXT: s_add_u32 s30, s16, 0x80
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s31
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s31, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s30
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s31
+; GCN-NOHSA-VI-NEXT: s_add_u32 s30, s16, 0x70
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s38
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s39
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s31, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s24
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s30
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s36
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s37
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s34
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s35
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s25
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s31
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s20
-; GCN-NOHSA-VI-NEXT: s_add_u32 s20, s16, 0x60
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s21
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s21, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s20
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s21
-; GCN-NOHSA-VI-NEXT: s_add_u32 s20, s16, 0x50
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s30
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s31
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s21, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s22
+; GCN-NOHSA-VI-NEXT: s_add_u32 s22, s16, 0x60
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s23
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s23, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s22
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s23
+; GCN-NOHSA-VI-NEXT: s_add_u32 s22, s16, 0x50
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s28
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s29
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s23, s17, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s20
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s28
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s26
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s27
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s21
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s22
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s26
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s27
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s24
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s25
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s23
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 64
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s16, 48
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s22
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s23
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s17, 0
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s14
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s15
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s12
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s13
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s14
+; GCN-NOHSA-VI-NEXT: s_add_u32 s14, s16, 64
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s15
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s15, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s20
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s21
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s14
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s15
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: s_nop 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NOHSA-VI-NEXT: s_add_u32 s0, s16, 32
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s1, s17, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s1
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s0
-; GCN-NOHSA-VI-NEXT: s_add_u32 s0, s16, 16
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s10
+; GCN-NOHSA-VI-NEXT: s_add_u32 s10, s16, 48
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s11
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s1, s17, 0
-; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s1
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s8
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s9
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s6
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s7
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s11, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s12
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s13
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s10
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s11
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s16
+; GCN-NOHSA-VI-NEXT: s_nop 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s8
+; GCN-NOHSA-VI-NEXT: s_add_u32 s8, s16, 32
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s9
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s9, s17, 0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s18
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s19
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s8
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s9
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-NOHSA-VI-NEXT: s_nop 0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s4
+; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s16, 16
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s17, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s7
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
+; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s16
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s17
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: s_endpgm
@@ -8917,89 +8913,83 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b512 s[0:15], s[18:19], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s28, s2, 16
-; GFX12-NEXT: s_lshr_b32 s42, s5, 16
-; GFX12-NEXT: s_lshr_b32 s50, s8, 16
-; GFX12-NEXT: s_mov_b32 s60, s11
-; GFX12-NEXT: s_lshr_b32 s22, s0, 16
-; GFX12-NEXT: s_mov_b32 s24, s1
-; GFX12-NEXT: s_lshr_b32 s26, s1, 16
-; GFX12-NEXT: s_mov_b32 s30, s3
-; GFX12-NEXT: s_lshr_b32 s36, s3, 16
-; GFX12-NEXT: s_lshr_b32 s38, s4, 16
-; GFX12-NEXT: s_mov_b32 s40, s5
-; GFX12-NEXT: s_lshr_b32 s44, s6, 16
-; GFX12-NEXT: s_mov_b32 s46, s7
-; GFX12-NEXT: s_lshr_b32 s48, s7, 16
-; GFX12-NEXT: s_mov_b32 s52, s9
-; GFX12-NEXT: s_lshr_b32 s54, s9, 16
-; GFX12-NEXT: s_bfe_i64 s[56:57], s[10:11], 0x100000
-; GFX12-NEXT: s_lshr_b32 s58, s10, 16
-; GFX12-NEXT: s_lshr_b32 s62, s11, 16
-; GFX12-NEXT: s_bfe_i64 s[10:11], s[28:29], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[28:29], s[42:43], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[42:43], s[50:51], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[50:51], s[60:61], 0x100000
-; GFX12-NEXT: s_lshr_b32 s60, s14, 16
-; GFX12-NEXT: s_bfe_i64 s[64:65], s[14:15], 0x100000
-; GFX12-NEXT: s_mov_b32 s14, s15
-; GFX12-NEXT: s_lshr_b32 s66, s15, 16
-; GFX12-NEXT: s_bfe_i64 s[18:19], s[0:1], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[20:21], s[6:7], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[34:35], s[8:9], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[22:23], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[24:25], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[26:27], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[24:25], s[30:31], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[22:23], s[36:37], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[26:27], s[38:39], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[30:31], s[40:41], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[36:37], s[44:45], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[40:41], s[46:47], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[38:39], s[48:49], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[44:45], s[52:53], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[46:47], s[54:55], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[48:49], s[58:59], 0x100000
-; GFX12-NEXT: s_lshr_b32 s52, s12, 16
-; GFX12-NEXT: s_bfe_i64 s[54:55], s[12:13], 0x100000
-; GFX12-NEXT: s_mov_b32 s12, s13
-; GFX12-NEXT: s_lshr_b32 s58, s13, 16
+; GFX12-NEXT: s_lshr_b32 s40, s15, 16
+; GFX12-NEXT: s_mov_b32 s42, s15
+; GFX12-NEXT: s_lshr_b32 s44, s14, 16
+; GFX12-NEXT: s_lshr_b32 s46, s13, 16
+; GFX12-NEXT: s_mov_b32 s48, s13
+; GFX12-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x100000
+; GFX12-NEXT: s_lshr_b32 s50, s12, 16
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x100000
-; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s15
+; GFX12-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s43
+; GFX12-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v0, s42 :: v_dual_mov_b32 v3, s41
+; GFX12-NEXT: v_dual_mov_b32 v2, s40 :: v_dual_mov_b32 v5, s15
+; GFX12-NEXT: s_lshr_b32 s36, s11, 16
+; GFX12-NEXT: s_mov_b32 s52, s11
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x100000
-; GFX12-NEXT: v_dual_mov_b32 v0, s14 :: v_dual_mov_b32 v3, s67
-; GFX12-NEXT: v_dual_mov_b32 v2, s66 :: v_dual_mov_b32 v5, s65
+; GFX12-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v4, s14 :: v_dual_mov_b32 v7, s45
+; GFX12-NEXT: v_dual_mov_b32 v6, s44 :: v_dual_mov_b32 v9, s49
+; GFX12-NEXT: s_lshr_b32 s54, s10, 16
+; GFX12-NEXT: v_dual_mov_b32 v8, s48 :: v_dual_mov_b32 v11, s47
+; GFX12-NEXT: v_dual_mov_b32 v10, s46 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: s_lshr_b32 s56, s9, 16
+; GFX12-NEXT: s_mov_b32 s58, s9
; GFX12-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x100000
-; GFX12-NEXT: v_dual_mov_b32 v4, s64 :: v_dual_mov_b32 v7, s61
-; GFX12-NEXT: v_dual_mov_b32 v6, s60 :: v_dual_mov_b32 v9, s13
-; GFX12-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s59
-; GFX12-NEXT: v_dual_mov_b32 v10, s58 :: v_dual_mov_b32 v13, s55
-; GFX12-NEXT: v_dual_mov_b32 v12, s54 :: v_dual_mov_b32 v15, s53
-; GFX12-NEXT: v_mov_b32_e32 v14, s52
-; GFX12-NEXT: s_bfe_i64 s[12:13], s[62:63], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v15, s51
+; GFX12-NEXT: v_mov_b32_e32 v14, s50
+; GFX12-NEXT: s_bfe_i64 s[12:13], s[36:37], 0x100000
+; GFX12-NEXT: s_lshr_b32 s60, s8, 16
+; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x100000
+; GFX12-NEXT: s_lshr_b32 s62, s7, 16
+; GFX12-NEXT: s_mov_b32 s64, s7
+; GFX12-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x100000
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[16:17] offset:240
; GFX12-NEXT: global_store_b128 v24, v[4:7], s[16:17] offset:224
; GFX12-NEXT: global_store_b128 v24, v[8:11], s[16:17] offset:208
; GFX12-NEXT: global_store_b128 v24, v[12:15], s[16:17] offset:192
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s50 :: v_dual_mov_b32 v3, s13
-; GFX12-NEXT: v_dual_mov_b32 v1, s51 :: v_dual_mov_b32 v2, s12
-; GFX12-NEXT: v_dual_mov_b32 v5, s57 :: v_dual_mov_b32 v4, s56
-; GFX12-NEXT: v_dual_mov_b32 v7, s49 :: v_dual_mov_b32 v6, s48
-; GFX12-NEXT: v_dual_mov_b32 v9, s45 :: v_dual_mov_b32 v8, s44
-; GFX12-NEXT: v_dual_mov_b32 v11, s47 :: v_dual_mov_b32 v10, s46
-; GFX12-NEXT: v_dual_mov_b32 v13, s35 :: v_dual_mov_b32 v12, s34
-; GFX12-NEXT: v_dual_mov_b32 v15, s43 :: v_dual_mov_b32 v14, s42
-; GFX12-NEXT: v_dual_mov_b32 v17, s41 :: v_dual_mov_b32 v16, s40
-; GFX12-NEXT: v_dual_mov_b32 v19, s39 :: v_dual_mov_b32 v18, s38
-; GFX12-NEXT: v_dual_mov_b32 v21, s21 :: v_dual_mov_b32 v20, s20
-; GFX12-NEXT: v_dual_mov_b32 v23, s37 :: v_dual_mov_b32 v22, s36
+; GFX12-NEXT: v_dual_mov_b32 v0, s52 :: v_dual_mov_b32 v3, s13
+; GFX12-NEXT: v_dual_mov_b32 v1, s53 :: v_dual_mov_b32 v2, s12
+; GFX12-NEXT: v_mov_b32_e32 v5, s11
+; GFX12-NEXT: s_lshr_b32 s38, s6, 16
+; GFX12-NEXT: s_lshr_b32 s28, s5, 16
+; GFX12-NEXT: s_mov_b32 s30, s5
+; GFX12-NEXT: s_bfe_i64 s[74:75], s[8:9], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v4, s10 :: v_dual_mov_b32 v7, s55
+; GFX12-NEXT: v_dual_mov_b32 v6, s54 :: v_dual_mov_b32 v9, s59
+; GFX12-NEXT: s_lshr_b32 s26, s4, 16
+; GFX12-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v8, s58 :: v_dual_mov_b32 v11, s57
+; GFX12-NEXT: v_dual_mov_b32 v10, s56 :: v_dual_mov_b32 v13, s75
+; GFX12-NEXT: s_lshr_b32 s22, s3, 16
+; GFX12-NEXT: s_mov_b32 s24, s3
+; GFX12-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v12, s74 :: v_dual_mov_b32 v15, s61
+; GFX12-NEXT: v_dual_mov_b32 v14, s60 :: v_dual_mov_b32 v17, s65
+; GFX12-NEXT: s_lshr_b32 s66, s2, 16
+; GFX12-NEXT: s_bfe_i64 s[20:21], s[4:5], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v16, s64 :: v_dual_mov_b32 v19, s63
+; GFX12-NEXT: v_dual_mov_b32 v18, s62 :: v_dual_mov_b32 v21, s35
+; GFX12-NEXT: s_lshr_b32 s68, s1, 16
+; GFX12-NEXT: s_mov_b32 s70, s1
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v20, s34 :: v_dual_mov_b32 v23, s39
+; GFX12-NEXT: v_mov_b32_e32 v22, s38
; GFX12-NEXT: s_clause 0x5
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[16:17] offset:176
; GFX12-NEXT: global_store_b128 v24, v[4:7], s[16:17] offset:160
@@ -9009,16 +8999,24 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: global_store_b128 v24, v[20:23], s[16:17] offset:96
; GFX12-NEXT: v_dual_mov_b32 v0, s30 :: v_dual_mov_b32 v3, s29
; GFX12-NEXT: v_dual_mov_b32 v1, s31 :: v_dual_mov_b32 v2, s28
-; GFX12-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
-; GFX12-NEXT: v_dual_mov_b32 v7, s27 :: v_dual_mov_b32 v6, s26
-; GFX12-NEXT: v_dual_mov_b32 v9, s25 :: v_dual_mov_b32 v8, s24
-; GFX12-NEXT: v_dual_mov_b32 v11, s23 :: v_dual_mov_b32 v10, s22
-; GFX12-NEXT: v_dual_mov_b32 v13, s1 :: v_dual_mov_b32 v12, s0
-; GFX12-NEXT: v_dual_mov_b32 v15, s11 :: v_dual_mov_b32 v14, s10
-; GFX12-NEXT: v_dual_mov_b32 v17, s9 :: v_dual_mov_b32 v16, s8
-; GFX12-NEXT: v_dual_mov_b32 v19, s7 :: v_dual_mov_b32 v18, s6
-; GFX12-NEXT: v_dual_mov_b32 v21, s19 :: v_dual_mov_b32 v20, s18
-; GFX12-NEXT: v_dual_mov_b32 v23, s5 :: v_dual_mov_b32 v22, s4
+; GFX12-NEXT: v_mov_b32_e32 v5, s21
+; GFX12-NEXT: s_lshr_b32 s72, s0, 16
+; GFX12-NEXT: s_bfe_i64 s[18:19], s[2:3], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[8:9], s[66:67], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v7, s27
+; GFX12-NEXT: v_dual_mov_b32 v6, s26 :: v_dual_mov_b32 v9, s25
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[70:71], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[68:69], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v11, s23
+; GFX12-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v13, s19
+; GFX12-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[72:73], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v15, s9
+; GFX12-NEXT: v_dual_mov_b32 v14, s8 :: v_dual_mov_b32 v17, s7
+; GFX12-NEXT: v_dual_mov_b32 v16, s6 :: v_dual_mov_b32 v19, s5
+; GFX12-NEXT: v_dual_mov_b32 v18, s4 :: v_dual_mov_b32 v21, s1
+; GFX12-NEXT: v_dual_mov_b32 v20, s0 :: v_dual_mov_b32 v23, s3
+; GFX12-NEXT: v_mov_b32_e32 v22, s2
; GFX12-NEXT: s_clause 0x5
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[16:17] offset:80
; GFX12-NEXT: global_store_b128 v24, v[4:7], s[16:17] offset:64
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 6e7185bd83442..f88b0c8d505bb 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -437,10 +437,10 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s2
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s3
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s1
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s0
; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0
; GFX6-NOHSA-NEXT: s_endpgm
@@ -455,21 +455,21 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX7-HSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GFX7-HSA-NEXT: s_add_u32 s10, s8, 16
; GFX7-HSA-NEXT: s_addc_u32 s11, s9, 0
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s10
-; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s11
+; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s10
+; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s11
; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s4
; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s5
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s6
; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-HSA-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s8
+; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s9
; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s3
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-HSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX7-HSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GFX7-HSA-NEXT: s_endpgm
;
; GFX8-NOHSA-LABEL: constant_load_v8i32:
@@ -479,21 +479,21 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX8-NOHSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GFX8-NOHSA-NEXT: s_add_u32 s10, s8, 16
; GFX8-NOHSA-NEXT: s_addc_u32 s11, s9, 0
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s10
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v7, s11
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v10, s10
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v11, s11
; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v8, s8
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v9, s9
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v7, s3
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GFX8-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: constant_load_v8i32:
@@ -526,10 +526,10 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX9-HSA-NEXT: v_mov_b32_e32 v1, s5
; GFX9-HSA-NEXT: v_mov_b32_e32 v2, s6
; GFX9-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-HSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-HSA-NEXT: v_mov_b32_e32 v6, s2
; GFX9-HSA-NEXT: v_mov_b32_e32 v7, s3
+; GFX9-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GFX9-HSA-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-HSA-NEXT: v_mov_b32_e32 v4, s0
; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] offset:16
; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9]
; GFX9-HSA-NEXT: s_endpgm
@@ -542,9 +542,9 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s1
-; GFX12-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v7, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, s2
+; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v7, s3
+; GFX12-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
+; GFX12-NEXT: v_mov_b32_e32 v4, s0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v8, v[0:3], s[8:9] offset:16
; GFX12-NEXT: global_store_b128 v8, v[4:7], s[8:9]
@@ -556,14 +556,14 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b256 s[0:7], s[10:11], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s4
-; GFX1250-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
-; GFX1250-NEXT: v_dual_mov_b32 v3, s7 :: v_dual_mov_b32 v4, s0
-; GFX1250-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v6, s2
-; GFX1250-NEXT: v_mov_b32_e32 v7, s3
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX1250-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[8:9] offset:16
; GFX1250-NEXT: global_store_b128 v8, v[4:7], s[8:9]
@@ -1361,23 +1361,22 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s13
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s14
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s15
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:48
-; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s7
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s8
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s9
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s10
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s11
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
-; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s7
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s6
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s5
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s4
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:48
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:32
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[16:19], 0 offset:16
+; GFX6-NOHSA-NEXT: s_waitcnt expcnt(2)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:32
; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; GFX6-NOHSA-NEXT: s_endpgm
;
@@ -1391,38 +1390,38 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX7-HSA-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX7-HSA-NEXT: s_add_u32 s18, s16, 48
; GFX7-HSA-NEXT: s_addc_u32 s19, s17, 0
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s18
-; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s19
+; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s18
+; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s19
+; GFX7-HSA-NEXT: s_add_u32 s18, s16, 32
+; GFX7-HSA-NEXT: s_addc_u32 s19, s17, 0
; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-HSA-NEXT: s_add_u32 s8, s16, 32
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s9
+; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s10
+; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s11
+; GFX7-HSA-NEXT: s_add_u32 s8, s16, 16
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s12
; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s13
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s14
; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s15
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s9
; GFX7-HSA-NEXT: s_addc_u32 s9, s17, 0
-; GFX7-HSA-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s10
-; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s11
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s18
+; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s19
+; GFX7-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX7-HSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-HSA-NEXT: s_add_u32 s4, s16, 16
+; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s8
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s9
; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-HSA-NEXT: s_addc_u32 s5, s17, 0
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s6
; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s5
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s16
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s17
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s0
; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s1
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s2
; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s17
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: s_endpgm
;
@@ -1433,38 +1432,38 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX8-NOHSA-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX8-NOHSA-NEXT: s_add_u32 s18, s16, 48
; GFX8-NOHSA-NEXT: s_addc_u32 s19, s17, 0
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s18
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v7, s19
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v8, s18
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v9, s19
+; GFX8-NOHSA-NEXT: s_add_u32 s18, s16, 32
+; GFX8-NOHSA-NEXT: s_addc_u32 s19, s17, 0
; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NOHSA-NEXT: s_add_u32 s8, s16, 32
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s9
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v7, s11
+; GFX8-NOHSA-NEXT: s_add_u32 s8, s16, 16
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s12
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s13
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s14
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s15
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NOHSA-NEXT: s_addc_u32 s9, s17, 0
-; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v6, s10
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v7, s11
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v10, s18
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v11, s19
+; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NOHSA-NEXT: s_add_u32 s4, s16, 16
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s8
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NOHSA-NEXT: s_addc_u32 s5, s17, 0
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s16
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s17
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s17
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NOHSA-NEXT: s_endpgm
;
@@ -1497,7 +1496,7 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX9-HSA-LABEL: constant_load_v16i32:
; GFX9-HSA: ; %bb.0: ; %entry
; GFX9-HSA-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x0
-; GFX9-HSA-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-HSA-NEXT: v_mov_b32_e32 v16, 0
; GFX9-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-HSA-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX9-HSA-NEXT: s_waitcnt lgkmcnt(0)
@@ -1509,19 +1508,18 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX9-HSA-NEXT: v_mov_b32_e32 v5, s9
; GFX9-HSA-NEXT: v_mov_b32_e32 v6, s10
; GFX9-HSA-NEXT: v_mov_b32_e32 v7, s11
-; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17] offset:48
-; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17] offset:32
-; GFX9-HSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-HSA-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-HSA-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17] offset:16
-; GFX9-HSA-NEXT: s_nop 0
-; GFX9-HSA-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-HSA-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-HSA-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-HSA-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX9-HSA-NEXT: v_mov_b32_e32 v11, s7
+; GFX9-HSA-NEXT: v_mov_b32_e32 v10, s6
+; GFX9-HSA-NEXT: v_mov_b32_e32 v9, s5
+; GFX9-HSA-NEXT: v_mov_b32_e32 v8, s4
+; GFX9-HSA-NEXT: v_mov_b32_e32 v15, s3
+; GFX9-HSA-NEXT: v_mov_b32_e32 v14, s2
+; GFX9-HSA-NEXT: v_mov_b32_e32 v13, s1
+; GFX9-HSA-NEXT: v_mov_b32_e32 v12, s0
+; GFX9-HSA-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; GFX9-HSA-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; GFX9-HSA-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; GFX9-HSA-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; GFX9-HSA-NEXT: s_endpgm
;
; GFX12-LABEL: constant_load_v16i32:
@@ -1534,11 +1532,11 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s15
; GFX12-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s9
; GFX12-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v9, s5
-; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v11, s7
-; GFX12-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v13, s1
-; GFX12-NEXT: v_dual_mov_b32 v12, s0 :: v_dual_mov_b32 v15, s3
-; GFX12-NEXT: v_mov_b32_e32 v14, s2
+; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s7
+; GFX12-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v9, s5
+; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v15, s3
+; GFX12-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v13, s1
+; GFX12-NEXT: v_mov_b32_e32 v12, s0
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[16:17] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[16:17] offset:32
@@ -1552,18 +1550,18 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b512 s[0:15], s[18:19], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s12
-; GFX1250-NEXT: v_dual_mov_b32 v1, s13 :: v_dual_mov_b32 v2, s14
-; GFX1250-NEXT: v_dual_mov_b32 v3, s15 :: v_dual_mov_b32 v4, s8
-; GFX1250-NEXT: v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s4
-; GFX1250-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v10, s6
-; GFX1250-NEXT: v_dual_mov_b32 v11, s7 :: v_dual_mov_b32 v12, s0
-; GFX1250-NEXT: v_dual_mov_b32 v13, s1 :: v_dual_mov_b32 v14, s2
-; GFX1250-NEXT: v_mov_b32_e32 v15, s3
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1250-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX1250-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX1250-NEXT: v_mov_b64_e32 v[8:9], s[4:5]
+; GFX1250-NEXT: v_mov_b64_e32 v[14:15], s[2:3]
+; GFX1250-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[16:17] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[16:17] offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
index 9b74de1c3eb60..3fe99930ee879 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
@@ -172,13 +172,12 @@ define amdgpu_kernel void @constant_load_v3i64(ptr addrspace(1) %out, ptr addrsp
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
-; GFX6-NEXT: v_mov_b32_e32 v3, s9
+; GFX6-NEXT: v_mov_b32_e32 v4, s8
+; GFX6-NEXT: v_mov_b32_e32 v5, s9
; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[2:3], off, s[0:3], 0 offset:16
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: v_mov_b32_e32 v3, s7
+; GFX6-NEXT: buffer_store_dwordx2 v[4:5], off, s[0:3], 0 offset:16
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
@@ -189,22 +188,22 @@ define amdgpu_kernel void @constant_load_v3i64(ptr addrspace(1) %out, ptr addrsp
; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dwordx2 s[8:9], s[2:3], 0x4
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX7-NEXT: s_add_u32 s2, s0, 16
-; GFX7-NEXT: s_addc_u32 s3, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x4
+; GFX7-NEXT: s_add_u32 s8, s0, 16
+; GFX7-NEXT: s_addc_u32 s9, s1, 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s8
+; GFX7-NEXT: v_mov_b32_e32 v7, s9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, s8
-; GFX7-NEXT: v_mov_b32_e32 v6, s9
+; GFX7-NEXT: v_mov_b32_e32 v9, s3
+; GFX7-NEXT: v_mov_b32_e32 v8, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mov_b32_e32 v2, s6
-; GFX7-NEXT: flat_store_dwordx2 v[3:4], v[5:6]
; GFX7-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_store_dwordx2 v[6:7], v[8:9]
; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: s_endpgm
;
@@ -212,22 +211,22 @@ define amdgpu_kernel void @constant_load_v3i64(ptr addrspace(1) %out, ptr addrsp
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx2 s[8:9], s[2:3], 0x10
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX8-NEXT: s_add_u32 s2, s0, 16
-; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x10
+; GFX8-NEXT: s_add_u32 s8, s0, 16
+; GFX8-NEXT: s_addc_u32 s9, s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s8
+; GFX8-NEXT: v_mov_b32_e32 v7, s9
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, s8
-; GFX8-NEXT: v_mov_b32_e32 v6, s9
+; GFX8-NEXT: v_mov_b32_e32 v9, s3
+; GFX8-NEXT: v_mov_b32_e32 v8, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[5:6]
; GFX8-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_store_dwordx2 v[6:7], v[8:9]
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
@@ -285,10 +284,10 @@ define amdgpu_kernel void @constant_load_v4i64(ptr addrspace(1) %out, ptr addrsp
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: v_mov_b32_e32 v3, s7
-; GFX6-NEXT: v_mov_b32_e32 v4, s0
-; GFX6-NEXT: v_mov_b32_e32 v5, s1
-; GFX6-NEXT: v_mov_b32_e32 v6, s2
; GFX6-NEXT: v_mov_b32_e32 v7, s3
+; GFX6-NEXT: v_mov_b32_e32 v6, s2
+; GFX6-NEXT: v_mov_b32_e32 v5, s1
+; GFX6-NEXT: v_mov_b32_e32 v4, s0
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GFX6-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0
; GFX6-NEXT: s_endpgm
@@ -303,21 +302,21 @@ define amdgpu_kernel void @constant_load_v4i64(ptr addrspace(1) %out, ptr addrsp
; GFX7-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GFX7-NEXT: s_add_u32 s10, s8, 16
; GFX7-NEXT: s_addc_u32 s11, s9, 0
-; GFX7-NEXT: v_mov_b32_e32 v6, s10
-; GFX7-NEXT: v_mov_b32_e32 v7, s11
+; GFX7-NEXT: v_mov_b32_e32 v10, s10
+; GFX7-NEXT: v_mov_b32_e32 v11, s11
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
+; GFX7-NEXT: v_mov_b32_e32 v8, s8
+; GFX7-NEXT: v_mov_b32_e32 v9, s9
; GFX7-NEXT: v_mov_b32_e32 v7, s3
-; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX7-NEXT: v_mov_b32_e32 v6, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX7-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: constant_load_v4i64:
@@ -327,21 +326,21 @@ define amdgpu_kernel void @constant_load_v4i64(ptr addrspace(1) %out, ptr addrsp
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
; GFX8-NEXT: s_add_u32 s10, s8, 16
; GFX8-NEXT: s_addc_u32 s11, s9, 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
-; GFX8-NEXT: v_mov_b32_e32 v7, s11
+; GFX8-NEXT: v_mov_b32_e32 v10, s10
+; GFX8-NEXT: v_mov_b32_e32 v11, s11
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NEXT: v_mov_b32_e32 v8, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s9
; GFX8-NEXT: v_mov_b32_e32 v7, s3
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GFX8-NEXT: s_endpgm
;
; EG-LABEL: constant_load_v4i64:
@@ -371,9 +370,9 @@ define amdgpu_kernel void @constant_load_v4i64(ptr addrspace(1) %out, ptr addrsp
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s1
-; GFX12-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v7, s3
-; GFX12-NEXT: v_mov_b32_e32 v6, s2
+; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v7, s3
+; GFX12-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
+; GFX12-NEXT: v_mov_b32_e32 v4, s0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v8, v[0:3], s[8:9] offset:16
; GFX12-NEXT: global_store_b128 v8, v[4:7], s[8:9]
@@ -397,23 +396,22 @@ define amdgpu_kernel void @constant_load_v8i64(ptr addrspace(1) %out, ptr addrsp
; GFX6-NEXT: v_mov_b32_e32 v1, s13
; GFX6-NEXT: v_mov_b32_e32 v2, s14
; GFX6-NEXT: v_mov_b32_e32 v3, s15
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:48
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_mov_b32_e32 v3, s7
; GFX6-NEXT: v_mov_b32_e32 v4, s8
; GFX6-NEXT: v_mov_b32_e32 v5, s9
; GFX6-NEXT: v_mov_b32_e32 v6, s10
; GFX6-NEXT: v_mov_b32_e32 v7, s11
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v11, s7
+; GFX6-NEXT: v_mov_b32_e32 v10, s6
+; GFX6-NEXT: v_mov_b32_e32 v9, s5
+; GFX6-NEXT: v_mov_b32_e32 v8, s4
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:48
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:32
+; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[16:19], 0 offset:16
+; GFX6-NEXT: s_waitcnt expcnt(2)
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:32
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; GFX6-NEXT: s_endpgm
;
@@ -427,38 +425,38 @@ define amdgpu_kernel void @constant_load_v8i64(ptr addrspace(1) %out, ptr addrsp
; GFX7-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX7-NEXT: s_add_u32 s18, s16, 48
; GFX7-NEXT: s_addc_u32 s19, s17, 0
-; GFX7-NEXT: v_mov_b32_e32 v6, s18
-; GFX7-NEXT: v_mov_b32_e32 v7, s19
+; GFX7-NEXT: v_mov_b32_e32 v8, s18
+; GFX7-NEXT: v_mov_b32_e32 v9, s19
+; GFX7-NEXT: s_add_u32 s18, s16, 32
+; GFX7-NEXT: s_addc_u32 s19, s17, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-NEXT: s_add_u32 s8, s16, 32
+; GFX7-NEXT: v_mov_b32_e32 v5, s9
+; GFX7-NEXT: v_mov_b32_e32 v6, s10
+; GFX7-NEXT: v_mov_b32_e32 v7, s11
+; GFX7-NEXT: s_add_u32 s8, s16, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s12
; GFX7-NEXT: v_mov_b32_e32 v1, s13
; GFX7-NEXT: v_mov_b32_e32 v2, s14
; GFX7-NEXT: v_mov_b32_e32 v3, s15
-; GFX7-NEXT: v_mov_b32_e32 v5, s9
; GFX7-NEXT: s_addc_u32 s9, s17, 0
-; GFX7-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX7-NEXT: v_mov_b32_e32 v6, s10
-; GFX7-NEXT: v_mov_b32_e32 v7, s11
-; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX7-NEXT: v_mov_b32_e32 v10, s18
+; GFX7-NEXT: v_mov_b32_e32 v11, s19
+; GFX7-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX7-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: s_add_u32 s4, s16, 16
+; GFX7-NEXT: v_mov_b32_e32 v4, s8
+; GFX7-NEXT: v_mov_b32_e32 v5, s9
; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: s_addc_u32 s5, s17, 0
; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: v_mov_b32_e32 v4, s16
+; GFX7-NEXT: v_mov_b32_e32 v5, s17
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_mov_b32_e32 v5, s17
; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: s_endpgm
;
@@ -469,38 +467,38 @@ define amdgpu_kernel void @constant_load_v8i64(ptr addrspace(1) %out, ptr addrsp
; GFX8-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX8-NEXT: s_add_u32 s18, s16, 48
; GFX8-NEXT: s_addc_u32 s19, s17, 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s18
-; GFX8-NEXT: v_mov_b32_e32 v7, s19
+; GFX8-NEXT: v_mov_b32_e32 v8, s18
+; GFX8-NEXT: v_mov_b32_e32 v9, s19
+; GFX8-NEXT: s_add_u32 s18, s16, 32
+; GFX8-NEXT: s_addc_u32 s19, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: s_add_u32 s8, s16, 32
+; GFX8-NEXT: v_mov_b32_e32 v5, s9
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mov_b32_e32 v7, s11
+; GFX8-NEXT: s_add_u32 s8, s16, 16
; GFX8-NEXT: v_mov_b32_e32 v0, s12
; GFX8-NEXT: v_mov_b32_e32 v1, s13
; GFX8-NEXT: v_mov_b32_e32 v2, s14
; GFX8-NEXT: v_mov_b32_e32 v3, s15
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NEXT: s_addc_u32 s9, s17, 0
-; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
-; GFX8-NEXT: v_mov_b32_e32 v6, s10
-; GFX8-NEXT: v_mov_b32_e32 v7, s11
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT: v_mov_b32_e32 v10, s18
+; GFX8-NEXT: v_mov_b32_e32 v11, s19
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: s_add_u32 s4, s16, 16
+; GFX8-NEXT: v_mov_b32_e32 v4, s8
+; GFX8-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_addc_u32 s5, s17, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: v_mov_b32_e32 v4, s16
+; GFX8-NEXT: v_mov_b32_e32 v5, s17
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v5, s17
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
@@ -540,11 +538,11 @@ define amdgpu_kernel void @constant_load_v8i64(ptr addrspace(1) %out, ptr addrsp
; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s15
; GFX12-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s9
; GFX12-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v9, s5
-; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v11, s7
-; GFX12-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v13, s1
-; GFX12-NEXT: v_dual_mov_b32 v12, s0 :: v_dual_mov_b32 v15, s3
-; GFX12-NEXT: v_mov_b32_e32 v14, s2
+; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s7
+; GFX12-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v9, s5
+; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v15, s3
+; GFX12-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v13, s1
+; GFX12-NEXT: v_mov_b32_e32 v12, s0
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[16:17] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[16:17] offset:32
@@ -573,26 +571,18 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: v_mov_b32_e32 v1, s29
; GFX6-NEXT: v_mov_b32_e32 v2, s30
; GFX6-NEXT: v_mov_b32_e32 v3, s31
+; GFX6-NEXT: v_mov_b32_e32 v4, s24
+; GFX6-NEXT: v_mov_b32_e32 v5, s25
+; GFX6-NEXT: v_mov_b32_e32 v6, s26
+; GFX6-NEXT: v_mov_b32_e32 v7, s27
+; GFX6-NEXT: v_mov_b32_e32 v8, s16
+; GFX6-NEXT: v_mov_b32_e32 v9, s17
+; GFX6-NEXT: v_mov_b32_e32 v10, s18
+; GFX6-NEXT: v_mov_b32_e32 v11, s19
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:112
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s24
-; GFX6-NEXT: v_mov_b32_e32 v1, s25
-; GFX6-NEXT: v_mov_b32_e32 v2, s26
-; GFX6-NEXT: v_mov_b32_e32 v3, s27
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:96
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s20
-; GFX6-NEXT: v_mov_b32_e32 v1, s21
-; GFX6-NEXT: v_mov_b32_e32 v2, s22
-; GFX6-NEXT: v_mov_b32_e32 v3, s23
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:80
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s16
-; GFX6-NEXT: v_mov_b32_e32 v1, s17
-; GFX6-NEXT: v_mov_b32_e32 v2, s18
-; GFX6-NEXT: v_mov_b32_e32 v3, s19
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:64
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], off, s[36:39], 0 offset:96
+; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[36:39], 0 offset:64
+; GFX6-NEXT: s_waitcnt expcnt(2)
; GFX6-NEXT: v_mov_b32_e32 v0, s12
; GFX6-NEXT: v_mov_b32_e32 v1, s13
; GFX6-NEXT: v_mov_b32_e32 v2, s14
@@ -605,17 +595,23 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: v_mov_b32_e32 v3, s11
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:32
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_mov_b32_e32 v3, s7
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:16
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s20
+; GFX6-NEXT: v_mov_b32_e32 v1, s21
+; GFX6-NEXT: v_mov_b32_e32 v2, s22
+; GFX6-NEXT: v_mov_b32_e32 v3, s23
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:80
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: v_mov_b32_e32 v1, s5
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_mov_b32_e32 v3, s7
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:16
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: constant_load_v16i64:
@@ -635,14 +631,14 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: v_mov_b32_e32 v5, s9
; GFX7-NEXT: v_mov_b32_e32 v6, s10
; GFX7-NEXT: v_mov_b32_e32 v7, s11
-; GFX7-NEXT: v_mov_b32_e32 v8, s4
-; GFX7-NEXT: v_mov_b32_e32 v9, s5
-; GFX7-NEXT: v_mov_b32_e32 v10, s6
-; GFX7-NEXT: v_mov_b32_e32 v11, s7
-; GFX7-NEXT: v_mov_b32_e32 v12, s0
-; GFX7-NEXT: v_mov_b32_e32 v13, s1
-; GFX7-NEXT: v_mov_b32_e32 v14, s2
-; GFX7-NEXT: v_mov_b32_e32 v15, s3
+; GFX7-NEXT: v_mov_b32_e32 v11, s3
+; GFX7-NEXT: v_mov_b32_e32 v10, s2
+; GFX7-NEXT: v_mov_b32_e32 v9, s1
+; GFX7-NEXT: v_mov_b32_e32 v8, s0
+; GFX7-NEXT: v_mov_b32_e32 v12, s4
+; GFX7-NEXT: v_mov_b32_e32 v13, s5
+; GFX7-NEXT: v_mov_b32_e32 v14, s6
+; GFX7-NEXT: v_mov_b32_e32 v15, s7
; GFX7-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0
; GFX7-NEXT: s_add_u32 s18, s16, 0x70
; GFX7-NEXT: s_addc_u32 s19, s17, 0
@@ -654,53 +650,52 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_nop 0
; GFX7-NEXT: v_mov_b32_e32 v0, s18
; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: s_add_u32 s18, s16, 0x50
+; GFX7-NEXT: s_add_u32 s18, s16, 64
; GFX7-NEXT: s_addc_u32 s19, s17, 0
; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GFX7-NEXT: v_mov_b32_e32 v0, s18
; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: s_add_u32 s18, s16, 64
+; GFX7-NEXT: s_add_u32 s18, s16, 48
; GFX7-NEXT: s_addc_u32 s19, s17, 0
; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX7-NEXT: v_mov_b32_e32 v0, s18
-; GFX7-NEXT: v_mov_b32_e32 v1, s19
-; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s12
-; GFX7-NEXT: s_add_u32 s12, s16, 48
; GFX7-NEXT: v_mov_b32_e32 v1, s13
-; GFX7-NEXT: s_addc_u32 s13, s17, 0
; GFX7-NEXT: v_mov_b32_e32 v2, s14
; GFX7-NEXT: v_mov_b32_e32 v3, s15
+; GFX7-NEXT: s_add_u32 s12, s16, 32
+; GFX7-NEXT: v_mov_b32_e32 v4, s18
+; GFX7-NEXT: v_mov_b32_e32 v5, s19
+; GFX7-NEXT: s_addc_u32 s13, s17, 0
+; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: v_mov_b32_e32 v4, s12
; GFX7-NEXT: v_mov_b32_e32 v5, s13
-; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX7-NEXT: s_nop 0
; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: s_add_u32 s8, s16, 32
; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_addc_u32 s9, s17, 0
; GFX7-NEXT: v_mov_b32_e32 v2, s10
; GFX7-NEXT: v_mov_b32_e32 v3, s11
-; GFX7-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-NEXT: v_mov_b32_e32 v5, s9
-; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX7-NEXT: s_nop 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: s_add_u32 s4, s16, 16
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: s_addc_u32 s5, s17, 0
-; GFX7-NEXT: v_mov_b32_e32 v2, s6
-; GFX7-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: v_mov_b32_e32 v4, s16
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: v_mov_b32_e32 v3, s3
+; GFX7-NEXT: s_add_u32 s0, s16, 0x50
; GFX7-NEXT: v_mov_b32_e32 v5, s17
+; GFX7-NEXT: s_addc_u32 s1, s17, 0
+; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX7-NEXT: s_nop 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: s_add_u32 s0, s16, 16
+; GFX7-NEXT: s_addc_u32 s1, s17, 0
+; GFX7-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
+; GFX7-NEXT: v_mov_b32_e32 v3, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT: s_endpgm
;
@@ -708,82 +703,78 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x40
; GFX8-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x0
+; GFX8-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x40
; GFX8-NEXT: s_add_u32 s34, s36, 0x70
; GFX8-NEXT: s_addc_u32 s35, s37, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, s34
-; GFX8-NEXT: v_mov_b32_e32 v6, s35
+; GFX8-NEXT: v_mov_b32_e32 v8, s34
+; GFX8-NEXT: v_mov_b32_e32 v9, s35
+; GFX8-NEXT: s_add_u32 s34, s36, 0x60
+; GFX8-NEXT: s_addc_u32 s35, s37, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s24
+; GFX8-NEXT: v_mov_b32_e32 v5, s25
+; GFX8-NEXT: v_mov_b32_e32 v6, s26
+; GFX8-NEXT: v_mov_b32_e32 v7, s27
+; GFX8-NEXT: s_add_u32 s24, s36, 64
; GFX8-NEXT: v_mov_b32_e32 v0, s28
; GFX8-NEXT: v_mov_b32_e32 v1, s29
; GFX8-NEXT: v_mov_b32_e32 v2, s30
; GFX8-NEXT: v_mov_b32_e32 v3, s31
-; GFX8-NEXT: v_mov_b32_e32 v4, s24
-; GFX8-NEXT: s_add_u32 s24, s36, 0x60
-; GFX8-NEXT: flat_store_dwordx4 v[5:6], v[0:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s25
; GFX8-NEXT: s_addc_u32 s25, s37, 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s26
-; GFX8-NEXT: v_mov_b32_e32 v7, s27
-; GFX8-NEXT: v_mov_b32_e32 v0, s24
-; GFX8-NEXT: v_mov_b32_e32 v1, s25
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: s_add_u32 s20, s36, 0x50
-; GFX8-NEXT: v_mov_b32_e32 v1, s21
-; GFX8-NEXT: s_addc_u32 s21, s37, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s22
-; GFX8-NEXT: v_mov_b32_e32 v3, s23
-; GFX8-NEXT: v_mov_b32_e32 v4, s20
-; GFX8-NEXT: v_mov_b32_e32 v5, s21
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: v_mov_b32_e32 v10, s34
+; GFX8-NEXT: v_mov_b32_e32 v11, s35
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: v_mov_b32_e32 v0, s16
-; GFX8-NEXT: s_add_u32 s16, s36, 64
; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: s_addc_u32 s17, s37, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s18
; GFX8-NEXT: v_mov_b32_e32 v3, s19
-; GFX8-NEXT: v_mov_b32_e32 v4, s16
-; GFX8-NEXT: v_mov_b32_e32 v5, s17
+; GFX8-NEXT: s_add_u32 s16, s36, 48
+; GFX8-NEXT: v_mov_b32_e32 v4, s24
+; GFX8-NEXT: v_mov_b32_e32 v5, s25
+; GFX8-NEXT: s_addc_u32 s17, s37, 0
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s16
; GFX8-NEXT: v_mov_b32_e32 v0, s12
-; GFX8-NEXT: s_add_u32 s12, s36, 48
; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: s_addc_u32 s13, s37, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s14
; GFX8-NEXT: v_mov_b32_e32 v3, s15
+; GFX8-NEXT: s_add_u32 s12, s36, 32
+; GFX8-NEXT: v_mov_b32_e32 v5, s17
+; GFX8-NEXT: s_addc_u32 s13, s37, 0
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: v_mov_b32_e32 v4, s12
; GFX8-NEXT: v_mov_b32_e32 v5, s13
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: s_nop 0
; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: s_add_u32 s8, s36, 32
; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_addc_u32 s9, s37, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s10
; GFX8-NEXT: v_mov_b32_e32 v3, s11
-; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: s_add_u32 s4, s36, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: s_addc_u32 s5, s37, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_mov_b32_e32 v3, s7
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: v_mov_b32_e32 v4, s36
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: s_add_u32 s0, s36, 0x50
; GFX8-NEXT: v_mov_b32_e32 v5, s37
+; GFX8-NEXT: s_addc_u32 s1, s37, 0
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: s_add_u32 s0, s36, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, s20
+; GFX8-NEXT: v_mov_b32_e32 v1, s21
+; GFX8-NEXT: v_mov_b32_e32 v2, s22
+; GFX8-NEXT: v_mov_b32_e32 v3, s23
+; GFX8-NEXT: s_addc_u32 s1, s37, 0
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
@@ -839,28 +830,28 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
; GFX12-NEXT: v_dual_mov_b32 v0, s28 :: v_dual_mov_b32 v3, s31
; GFX12-NEXT: v_dual_mov_b32 v2, s30 :: v_dual_mov_b32 v5, s25
; GFX12-NEXT: v_dual_mov_b32 v4, s24 :: v_dual_mov_b32 v7, s27
-; GFX12-NEXT: v_dual_mov_b32 v6, s26 :: v_dual_mov_b32 v9, s21
-; GFX12-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v11, s23
-; GFX12-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v13, s17
-; GFX12-NEXT: v_dual_mov_b32 v12, s16 :: v_dual_mov_b32 v15, s19
-; GFX12-NEXT: v_dual_mov_b32 v14, s18 :: v_dual_mov_b32 v17, s13
-; GFX12-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v19, s15
-; GFX12-NEXT: v_dual_mov_b32 v18, s14 :: v_dual_mov_b32 v21, s9
-; GFX12-NEXT: v_dual_mov_b32 v20, s8 :: v_dual_mov_b32 v23, s11
-; GFX12-NEXT: v_dual_mov_b32 v22, s10 :: v_dual_mov_b32 v25, s5
-; GFX12-NEXT: v_dual_mov_b32 v24, s4 :: v_dual_mov_b32 v27, s7
-; GFX12-NEXT: v_dual_mov_b32 v26, s6 :: v_dual_mov_b32 v29, s1
-; GFX12-NEXT: v_dual_mov_b32 v28, s0 :: v_dual_mov_b32 v31, s3
-; GFX12-NEXT: v_mov_b32_e32 v30, s2
+; GFX12-NEXT: v_dual_mov_b32 v6, s26 :: v_dual_mov_b32 v9, s17
+; GFX12-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v19, s11
+; GFX12-NEXT: v_dual_mov_b32 v18, s10 :: v_dual_mov_b32 v17, s9
+; GFX12-NEXT: v_dual_mov_b32 v16, s8 :: v_dual_mov_b32 v23, s3
+; GFX12-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v11, s19
+; GFX12-NEXT: v_dual_mov_b32 v22, s2 :: v_dual_mov_b32 v21, s1
+; GFX12-NEXT: v_dual_mov_b32 v20, s0 :: v_dual_mov_b32 v25, s21
+; GFX12-NEXT: v_dual_mov_b32 v24, s20 :: v_dual_mov_b32 v27, s23
+; GFX12-NEXT: v_dual_mov_b32 v26, s22 :: v_dual_mov_b32 v29, s5
+; GFX12-NEXT: v_dual_mov_b32 v28, s4 :: v_dual_mov_b32 v31, s7
+; GFX12-NEXT: v_mov_b32_e32 v30, s6
; GFX12-NEXT: s_clause 0x7
; GFX12-NEXT: global_store_b128 v32, v[0:3], s[36:37] offset:112
; GFX12-NEXT: global_store_b128 v32, v[4:7], s[36:37] offset:96
-; GFX12-NEXT: global_store_b128 v32, v[8:11], s[36:37] offset:80
-; GFX12-NEXT: global_store_b128 v32, v[12:15], s[36:37] offset:64
-; GFX12-NEXT: global_store_b128 v32, v[16:19], s[36:37] offset:48
-; GFX12-NEXT: global_store_b128 v32, v[20:23], s[36:37] offset:32
-; GFX12-NEXT: global_store_b128 v32, v[24:27], s[36:37] offset:16
-; GFX12-NEXT: global_store_b128 v32, v[28:31], s[36:37]
+; GFX12-NEXT: global_store_b128 v32, v[12:15], s[36:37] offset:48
+; GFX12-NEXT: global_store_b128 v32, v[16:19], s[36:37] offset:32
+; GFX12-NEXT: global_store_b128 v32, v[8:11], s[36:37] offset:64
+; GFX12-NEXT: global_store_b128 v32, v[24:27], s[36:37] offset:80
+; GFX12-NEXT: global_store_b128 v32, v[20:23], s[36:37]
+; GFX12-NEXT: global_store_b128 v32, v[28:31], s[36:37] offset:16
; GFX12-NEXT: s_endpgm
entry:
%ld = load <16 x i64>, ptr addrspace(4) %in
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
index 8d7eb6dcb843c..aef724322228f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
@@ -259,12 +259,12 @@ define amdgpu_kernel void @global_load_v8f32(ptr addrspace(1) %out, ptr addrspac
; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: s_add_u32 s4, s2, 16
-; GCN-HSA-NEXT: s_addc_u32 s5, s3, 0
+; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT: s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT: s_addc_u32 s3, s3, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v4, s4
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, s5
; GCN-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GCN-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCN-HSA-NEXT: v_mov_b32_e32 v9, s1
@@ -274,9 +274,9 @@ define amdgpu_kernel void @global_load_v8f32(ptr addrspace(1) %out, ptr addrspac
; GCN-HSA-NEXT: v_mov_b32_e32 v11, s1
; GCN-HSA-NEXT: v_mov_b32_e32 v10, s0
; GCN-HSA-NEXT: s_waitcnt vmcnt(1)
-; GCN-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
; GCN-HSA-NEXT: s_waitcnt vmcnt(1)
-; GCN-HSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCN-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_load_v8f32:
@@ -315,17 +315,17 @@ define amdgpu_kernel void @global_load_v9f32(ptr addrspace(1) %out, ptr addrspac
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; SI-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
; SI-NOHSA-NEXT: s_endpgm
;
; GCN-HSA-LABEL: global_load_v9f32:
@@ -335,24 +335,24 @@ define amdgpu_kernel void @global_load_v9f32(ptr addrspace(1) %out, ptr addrspac
; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: s_add_u32 s4, s2, 16
+; GCN-HSA-NEXT: s_add_u32 s4, s2, 32
; GCN-HSA-NEXT: s_addc_u32 s5, s3, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCN-HSA-NEXT: s_add_u32 s2, s2, 32
+; GCN-HSA-NEXT: s_add_u32 s2, s2, 16
; GCN-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v4, s4
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v7, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v6, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCN-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GCN-HSA-NEXT: flat_load_dword v14, v[6:7]
+; GCN-HSA-NEXT: v_mov_b32_e32 v9, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, s4
; GCN-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT: flat_load_dword v14, v[8:9]
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 16
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v9, s1
; GCN-HSA-NEXT: v_mov_b32_e32 v8, s0
-; GCN-HSA-NEXT: s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT: s_add_u32 s0, s0, 32
; GCN-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v11, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v10, s2
@@ -361,9 +361,9 @@ define amdgpu_kernel void @global_load_v9f32(ptr addrspace(1) %out, ptr addrspac
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
; GCN-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: flat_store_dword v[10:11], v14
+; GCN-HSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GCN-HSA-NEXT: flat_store_dword v[12:13], v14
; GCN-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_load_v9f32:
@@ -376,17 +376,17 @@ define amdgpu_kernel void @global_load_v9f32(ptr addrspace(1) %out, ptr addrspac
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
-; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; GCNX3-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
; GCNX3-NOHSA-NEXT: s_endpgm
entry:
%tmp0 = load <9 x float>, ptr addrspace(1) %in
@@ -495,18 +495,18 @@ define amdgpu_kernel void @global_load_v11f32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:32
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[3:6], off, s[8:11], 0
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dword v2, off, s[4:7], 0 offset:40
-; SI-NOHSA-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 offset:32
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; SI-NOHSA-NEXT: buffer_store_dword v10, off, s[4:7], 0 offset:40
+; SI-NOHSA-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 offset:32
; SI-NOHSA-NEXT: s_endpgm
;
; GCN-HSA-LABEL: global_load_v11f32:
@@ -701,43 +701,44 @@ define amdgpu_kernel void @global_load_v16f32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCN-HSA-NEXT: s_add_u32 s4, s2, 16
; GCN-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCN-HSA-NEXT: s_add_u32 s6, s2, 48
-; GCN-HSA-NEXT: s_addc_u32 s7, s3, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v9, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v8, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v13, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v12, s4
+; GCN-HSA-NEXT: s_add_u32 s4, s2, 48
+; GCN-HSA-NEXT: s_addc_u32 s5, s3, 0
+; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCN-HSA-NEXT: s_add_u32 s2, s2, 32
; GCN-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v0, s6
-; GCN-HSA-NEXT: v_mov_b32_e32 v1, s7
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCN-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, s4
+; GCN-HSA-NEXT: v_mov_b32_e32 v5, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v9, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, s2
; GCN-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCN-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; GCN-HSA-NEXT: v_mov_b32_e32 v13, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v12, s4
; GCN-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GCN-HSA-NEXT: s_add_u32 s2, s0, 32
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 48
-; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT: v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v17, s1
; GCN-HSA-NEXT: v_mov_b32_e32 v16, s0
; GCN-HSA-NEXT: s_add_u32 s0, s0, 16
-; GCN-HSA-NEXT: v_mov_b32_e32 v19, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v18, s4
; GCN-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
-; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[0:3]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[16:17], v[0:3]
; GCN-HSA-NEXT: s_nop 0
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v3, s1
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s0
-; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
+; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
; GCN-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
+; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[8:11]
+; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
; GCN-HSA-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
; GCN-HSA-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index b150958ddd51d..dbfb90c1dad8d 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -7491,38 +7491,38 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[13:16], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[17:20], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[21:24], off, s[8:11], 0
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v17
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 16, v14
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v17, 0xffff, v14
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v21, 0xffff, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 16, v16
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v25, 0xffff, v16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v30, 16, v15
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xffff, v15
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[13:16], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v16, 16, v23
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v9, 16, v21
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v7, 0xffff, v21
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v14, 0xffff, v23
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v21, 0xffff, v18
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v3, 0xffff, v17
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 16, v20
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v25, 0xffff, v20
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xffff, v19
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[17:20], off, s[8:11], 0 offset:32
; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[31:34], off, s[8:11], 0 offset:48
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v7
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v3, 0xffff, v7
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v8, 16, v10
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v12, 16, v9
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v10, 0xffff, v9
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v10, 0xffff, v24
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v12, 16, v24
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v22
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v22
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v40, 16, v13
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v38, 0xffff, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v46, 16, v15
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xffff, v15
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v40, 16, v17
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v38, 0xffff, v17
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v46, 16, v19
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xffff, v19
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 16, v32
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v13, 0xffff, v32
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 16, v32
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v17, 0xffff, v32
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v52, 16, v34
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v50, 0xffff, v34
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v34, 16, v33
@@ -7538,8 +7538,8 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[47:50], off, s[0:3], 0 offset:192
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v45, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v47, v33
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 16, v16
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v41, 0xffff, v16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 16, v20
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v41, 0xffff, v20
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:160
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v42, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v44, v33
@@ -7548,39 +7548,38 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v41, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v29, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v31, v33
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v14
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v35, 0xffff, v14
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v14, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v16, v33
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v18
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v35, 0xffff, v18
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v18, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v20, v33
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[38:41], off, s[0:3], 0 offset:128
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v36, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v38, v33
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v26, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v28, v33
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:208
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[35:38], off, s[0:3], 0 offset:144
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v22, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v35, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v6, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v22, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v24, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v18, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v20, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v15, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v17, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v11, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v13, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v7, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v9, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, v33
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v8, v33
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:112
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:48
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, v33
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v6, v33
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0
-; GCN-NOHSA-VI-NEXT: s_nop 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v10, v33
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, v33
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -8322,111 +8321,112 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[15:18], off, s[8:11], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(3)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(2)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v17, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v22, 31, v21
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v18, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:240
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(2)
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v4, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v15, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v16, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:208
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v20, 16, v9
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v13, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v14
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v14, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:176
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v18, v9, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v11, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v12
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v9, v5, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v16, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v11, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v12, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v8, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v8, v7, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v10, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 16, v10
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v22, 16, v14
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v22, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v14, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 16, v15
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 31, v24
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v21, 16, v13
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v26, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v15, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 31, v24
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 16, v12
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v21, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v12, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v13, 0, 16
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(4)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v20, 16, v10
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v10, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v20, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v11, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v19, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v18, 16, v8
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v17, 16, v9
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v18, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v8, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v27, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(6)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v16, 16, v2
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 31, v24
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v11, v9, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v17, 0, 16
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(7)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v22, 16, v4
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v28, v4, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v6, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:144
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v19, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v7, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v20, v20, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v16, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v2, 0, 16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v22, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v22, 31, v21
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v28, 16, v0
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 16, v1
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v24, 16, v7
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 16, v6
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v1, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v25, 16, v5
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v16, v24, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v26, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v28, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v26, v3, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v28, v1, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v5, v5, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v4, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v7, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v18, v6, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v7, v25, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v20, v23, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v27, 31, v26
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v29, 31, v28
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v8, 31, v7
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 31, v22
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 31, v24
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v27, 31, v26
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[26:29], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[26:29], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:80
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[5:8], off, s[0:3], 0 offset:16
; GCN-NOHSA-VI-NEXT: s_endpgm
;
; EG-LABEL: global_sextload_v32i16_to_v32i64:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
index 930fe409c2cea..717c2cd65eeae 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -454,17 +454,17 @@ define amdgpu_kernel void @global_load_v9i32(ptr addrspace(1) %out, ptr addrspac
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; SI-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
; SI-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; SI-NOHSA-NEXT: s_endpgm
;
; GCNX3-HSA-LABEL: global_load_v9i32:
@@ -476,33 +476,33 @@ define amdgpu_kernel void @global_load_v9i32(ptr addrspace(1) %out, ptr addrspac
; GCNX3-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 32
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
-; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
+; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
+; GCNX3-HSA-NEXT: flat_load_dword v14, v[0:1]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s4
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GCNX3-HSA-NEXT: flat_load_dword v14, v[8:9]
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s0
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 32
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, s2
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GCNX3-HSA-NEXT: flat_store_dword v[12:13], v14
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dword v[12:13], v14
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCNX3-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_load_v9i32:
@@ -515,17 +515,17 @@ define amdgpu_kernel void @global_load_v9i32(ptr addrspace(1) %out, ptr addrspac
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: buffer_load_dword v8, off, s[8:11], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: buffer_store_dword v8, off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; GCNX3-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: global_load_v9i32:
@@ -558,15 +558,15 @@ define amdgpu_kernel void @global_load_v9i32(ptr addrspace(1) %out, ptr addrspac
; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-HSA-NEXT: v_mov_b32_e32 v8, 0
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v8, s[2:3]
-; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3] offset:16
+; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v8, s[2:3] offset:16
; GCN-HSA-NEXT: global_load_dword v9, v8, s[2:3] offset:32
+; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
-; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GCN-HSA-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
; GCN-HSA-NEXT: global_store_dword v8, v9, s[0:1] offset:32
+; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
+; GCN-HSA-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GCN-HSA-NEXT: s_endpgm
entry:
%ld = load <9 x i32>, ptr addrspace(1) %in
@@ -585,17 +585,17 @@ define amdgpu_kernel void @global_load_v10i32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; SI-NOHSA-NEXT: buffer_load_dwordx2 v[8:9], off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
; SI-NOHSA-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; SI-NOHSA-NEXT: s_endpgm
;
; GCNX3-HSA-LABEL: global_load_v10i32:
@@ -607,33 +607,33 @@ define amdgpu_kernel void @global_load_v10i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 32
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
-; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
+; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
+; GCNX3-HSA-NEXT: flat_load_dwordx2 v[8:9], v[0:1]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GCNX3-HSA-NEXT: flat_load_dwordx2 v[8:9], v[8:9]
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, s0
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 32
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, s0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s2
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GCNX3-HSA-NEXT: flat_store_dwordx2 v[14:15], v[8:9]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx2 v[14:15], v[8:9]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GCNX3-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_load_v10i32:
@@ -646,17 +646,17 @@ define amdgpu_kernel void @global_load_v10i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: buffer_load_dwordx2 v[8:9], off, s[8:11], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; GCNX3-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: global_load_v10i32:
@@ -687,15 +687,15 @@ define amdgpu_kernel void @global_load_v10i32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-HSA-NEXT: v_mov_b32_e32 v10, 0
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v10, s[2:3]
-; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v10, s[2:3] offset:16
+; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v10, s[2:3] offset:16
; GCN-HSA-NEXT: global_load_dwordx2 v[8:9], v10, s[2:3] offset:32
+; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v10, s[2:3]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
-; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v10, v[4:7], s[0:1] offset:16
+; GCN-HSA-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1] offset:16
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
; GCN-HSA-NEXT: global_store_dwordx2 v10, v[8:9], s[0:1] offset:32
+; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
+; GCN-HSA-NEXT: global_store_dwordx4 v10, v[4:7], s[0:1]
; GCN-HSA-NEXT: s_endpgm
entry:
%ld = load <10 x i32>, ptr addrspace(1) %in
@@ -714,18 +714,18 @@ define amdgpu_kernel void @global_load_v11i32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dword v10, off, s[4:7], 0 offset:40
-; SI-NOHSA-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: buffer_store_dword v6, off, s[4:7], 0 offset:40
+; SI-NOHSA-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
; SI-NOHSA-NEXT: s_endpgm
;
; GCNX3-HSA-LABEL: global_load_v11i32:
@@ -776,15 +776,15 @@ define amdgpu_kernel void @global_load_v11i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx3 v[8:10], off, s[8:11], 0 offset:32
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx3 v[8:10], off, s[4:7], 0 offset:32
; GCNX3-NOHSA-NEXT: s_endpgm
@@ -820,13 +820,13 @@ define amdgpu_kernel void @global_load_v11i32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-HSA-NEXT: v_mov_b32_e32 v11, 0
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v11, s[2:3]
-; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v11, s[2:3] offset:16
+; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v11, s[2:3] offset:16
+; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v11, s[2:3]
; GCN-HSA-NEXT: global_load_dwordx3 v[8:10], v11, s[2:3] offset:32
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v11, v[0:3], s[0:1]
+; GCN-HSA-NEXT: global_store_dwordx4 v11, v[0:3], s[0:1] offset:16
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v11, v[4:7], s[0:1] offset:16
+; GCN-HSA-NEXT: global_store_dwordx4 v11, v[4:7], s[0:1]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
; GCN-HSA-NEXT: global_store_dwordx3 v11, v[8:10], s[0:1] offset:32
; GCN-HSA-NEXT: s_endpgm
@@ -848,17 +848,17 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:32
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:32
; SI-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; SI-NOHSA-NEXT: s_endpgm
;
; GCNX3-HSA-LABEL: global_load_v12i32:
@@ -870,12 +870,12 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 32
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
@@ -887,16 +887,16 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s0
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 32
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v17, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, s2
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[0:3]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
; GCNX3-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_load_v12i32:
@@ -909,17 +909,17 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:32
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; GCNX3-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: global_load_v12i32:
@@ -951,15 +951,15 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-HSA-NEXT: v_mov_b32_e32 v12, 0
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v12, s[2:3]
-; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v12, s[2:3] offset:16
-; GCN-HSA-NEXT: global_load_dwordx4 v[8:11], v12, s[2:3] offset:32
+; GCN-HSA-NEXT: global_load_dwordx4 v[0:3], v12, s[2:3] offset:16
+; GCN-HSA-NEXT: global_load_dwordx4 v[4:7], v12, s[2:3] offset:32
+; GCN-HSA-NEXT: global_load_dwordx4 v[8:11], v12, s[2:3]
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1]
+; GCN-HSA-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v12, v[4:7], s[0:1] offset:16
+; GCN-HSA-NEXT: global_store_dwordx4 v12, v[4:7], s[0:1] offset:32
; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
-; GCN-HSA-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:32
+; GCN-HSA-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1]
; GCN-HSA-NEXT: s_endpgm
entry:
%ld = load <12 x i32>, ptr addrspace(1) %in
@@ -1003,21 +1003,21 @@ define amdgpu_kernel void @global_load_v16i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 16
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s4
-; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 48
-; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 32
-; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s5
+; GCNX3-HSA-NEXT: s_add_u32 s6, s2, 48
+; GCNX3-HSA-NEXT: s_addc_u32 s7, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
+; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 32
+; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s6
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s7
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s5
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s4
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 32
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
@@ -1028,17 +1028,16 @@ define amdgpu_kernel void @global_load_v16i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: v_mov_b32_e32 v17, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s0
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 16
-; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v21, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v20, s2
+; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v23, s1
; GCNX3-HSA-NEXT: v_mov_b32_e32 v22, s0
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[0:3]
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[20:21], v[4:7]
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[18:19], v[8:11]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[20:21], v[0:3]
+; GCNX3-HSA-NEXT: s_waitcnt vmcnt(2)
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[18:19], v[4:7]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(3)
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[22:23], v[12:15]
; GCNX3-HSA-NEXT: s_endpgm
@@ -4353,14 +4352,14 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; SI-NOHSA-LABEL: global_load_v32i32:
; SI-NOHSA: ; %bb.0:
-; SI-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; SI-NOHSA-NEXT: s_mov_b32 s6, -1
-; SI-NOHSA-NEXT: s_mov_b32 s10, s6
-; SI-NOHSA-NEXT: s_mov_b32 s11, s7
+; SI-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; SI-NOHSA-NEXT: s_mov_b32 s2, -1
+; SI-NOHSA-NEXT: s_mov_b32 s10, s2
+; SI-NOHSA-NEXT: s_mov_b32 s11, s3
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NOHSA-NEXT: s_mov_b32 s8, s2
-; SI-NOHSA-NEXT: s_mov_b32 s9, s3
+; SI-NOHSA-NEXT: s_mov_b32 s8, s6
+; SI-NOHSA-NEXT: s_mov_b32 s9, s7
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:96
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
@@ -4369,22 +4368,22 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:16
-; SI-NOHSA-NEXT: s_mov_b32 s4, s0
-; SI-NOHSA-NEXT: s_mov_b32 s5, s1
+; SI-NOHSA-NEXT: s_mov_b32 s0, s4
+; SI-NOHSA-NEXT: s_mov_b32 s1, s5
; SI-NOHSA-NEXT: s_waitcnt vmcnt(6)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:96
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:112
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
; SI-NOHSA-NEXT: s_waitcnt vmcnt(6)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:64
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:80
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:64
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:80
; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[20:23], off, s[4:7], 0 offset:48
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[24:27], off, s[4:7], 0
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0
; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:16
; SI-NOHSA-NEXT: s_endpgm
;
; GCNX3-HSA-LABEL: global_load_v32i32:
@@ -4417,16 +4416,16 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s4
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 0x70
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 0x60
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s6
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s7
-; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
+; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 0x60
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v21, s5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v20, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v25, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v24, s2
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
@@ -4460,11 +4459,11 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, s8
; GCNX3-HSA-NEXT: v_mov_b32_e32 v7, s9
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v3, s5
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[6:7], v[8:11]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, s6
; GCNX3-HSA-NEXT: v_mov_b32_e32 v7, s7
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v3, s5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s1
@@ -4483,14 +4482,14 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
;
; GCNX3-NOHSA-LABEL: global_load_v32i32:
; GCNX3-NOHSA: ; %bb.0:
-; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT: s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT: s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s3
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s7
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:96
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
@@ -4499,22 +4498,22 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:16
-; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT: s_mov_b32 s1, s5
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(6)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:96
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:112
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(6)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:64
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:80
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:64
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:80
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[20:23], off, s[4:7], 0 offset:48
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[24:27], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:16
; GCNX3-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: global_load_v32i32:
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index 43196c3d08440..c240876ad0da5 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -5572,15 +5572,15 @@ define amdgpu_kernel void @local_zextload_v4i16_to_v4i64(ptr addrspace(3) %out,
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s1
; SI-NEXT: ds_read_b64 v[0:1], v0
-; SI-NEXT: v_mov_b32_e32 v10, s0
-; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, s0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; SI-NEXT: ds_write2_b64 v10, v[4:5], v[2:3] offset0:2 offset1:3
-; SI-NEXT: ds_write2_b64 v10, v[6:7], v[8:9] offset1:1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; SI-NEXT: v_mov_b32_e32 v1, v3
+; SI-NEXT: ds_write2_b64 v8, v[6:7], v[2:3] offset0:2 offset1:3
+; SI-NEXT: ds_write2_b64 v8, v[0:1], v[4:5] offset1:1
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v4i16_to_v4i64:
@@ -5882,30 +5882,30 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: v_mov_b32_e32 v5, 0
+; SI-NEXT: v_mov_b32_e32 v11, v5
; SI-NEXT: v_mov_b32_e32 v7, v5
-; SI-NEXT: v_mov_b32_e32 v12, v5
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s1
; SI-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
-; SI-NEXT: v_mov_b32_e32 v13, s0
+; SI-NEXT: v_mov_b32_e32 v14, s0
; SI-NEXT: v_mov_b32_e32 v9, v5
+; SI-NEXT: v_mov_b32_e32 v13, v5
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: ds_write2_b64 v13, v[2:3], v[6:7] offset0:6 offset1:7
+; SI-NEXT: ds_write2_b64 v14, v[2:3], v[10:11] offset0:4 offset1:5
; SI-NEXT: v_mov_b32_e32 v2, v5
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: ds_write2_b64 v13, v[1:2], v[4:5] offset0:2 offset1:3
+; SI-NEXT: ds_write2_b64 v14, v[1:2], v[4:5] offset0:2 offset1:3
; SI-NEXT: v_mov_b32_e32 v1, v5
-; SI-NEXT: ds_write2_b64 v13, v[11:12], v[8:9] offset0:4 offset1:5
-; SI-NEXT: v_mov_b32_e32 v11, v5
-; SI-NEXT: ds_write2_b64 v13, v[0:1], v[10:11] offset1:1
+; SI-NEXT: ds_write2_b64 v14, v[12:13], v[6:7] offset0:6 offset1:7
+; SI-NEXT: ds_write2_b64 v14, v[0:1], v[8:9] offset1:1
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v8i16_to_v8i64:
@@ -6393,133 +6393,129 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; SI-NEXT: v_mov_b32_e32 v13, v9
; SI-NEXT: v_mov_b32_e32 v11, v9
; SI-NEXT: s_waitcnt lgkmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v4
+; SI-NEXT: ds_write2_b64 v18, v[16:17], v[14:15] offset0:8 offset1:9
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; SI-NEXT: v_mov_b32_e32 v5, v9
+; SI-NEXT: ds_write2_b64 v18, v[4:5], v[14:15] offset0:10 offset1:11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v6
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v7
-; SI-NEXT: ds_write2_b64 v18, v[16:17], v[14:15] offset0:10 offset1:11
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v7
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:14 offset1:15
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
+; SI-NEXT: ds_write2_b64 v18, v[14:15], v[4:5] offset0:12 offset1:13
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v7
+; SI-NEXT: v_mov_b32_e32 v6, v9
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: ds_write2_b64 v18, v[5:6], v[12:13] offset0:14 offset1:15
+; SI-NEXT: v_mov_b32_e32 v5, v9
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v3
+; SI-NEXT: v_mov_b32_e32 v3, v9
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v2
+; SI-NEXT: ds_write2_b64 v18, v[2:3], v[4:5] offset0:4 offset1:5
; SI-NEXT: v_mov_b32_e32 v2, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; SI-NEXT: ds_write2_b64 v18, v[12:13], v[10:11] offset0:6 offset1:7
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v6
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v4
+; SI-NEXT: v_mov_b32_e32 v7, v9
; SI-NEXT: ds_write2_b64 v18, v[1:2], v[8:9] offset0:2 offset1:3
-; SI-NEXT: v_mov_b32_e32 v4, v9
; SI-NEXT: v_mov_b32_e32 v1, v9
-; SI-NEXT: v_mov_b32_e32 v6, v9
-; SI-NEXT: v_mov_b32_e32 v8, v9
-; SI-NEXT: ds_write2_b64 v18, v[16:17], v[10:11] offset0:8 offset1:9
-; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:12 offset1:13
-; SI-NEXT: ds_write2_b64 v18, v[3:4], v[5:6] offset0:4 offset1:5
-; SI-NEXT: ds_write2_b64 v18, v[0:1], v[7:8] offset1:1
+; SI-NEXT: ds_write2_b64 v18, v[6:7], v[10:11] offset0:6 offset1:7
+; SI-NEXT: ds_write2_b64 v18, v[0:1], v[14:15] offset1:1
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v16i16_to_v16i64:
; VI-NO-DS128: ; %bb.0:
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, 0
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v8
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, 0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v15, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v17, v9
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, s0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v18, s0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v9
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[12:13], v[9:10] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[5:6], v[9:10] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v7
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[5:6], v[9:10] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; VI-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v4
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, v8
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[12:13], v[7:8] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[3:4], v[9:10] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v8
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; VI-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[2:3], v[6:7] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[1:2], v[5:6] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v12, v8
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[0:1], v[11:12] offset1:1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v4
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v4
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[14:15], v[16:17] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v5
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[14:15], v[4:5] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[14:15] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[6:7] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v9
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v0
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[2:3], v[6:7] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[14:15] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[0:1], v[12:13] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[10:11], v[8:9] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v16i16_to_v16i64:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, 0
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v8
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v17, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v9
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, s0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v18, s0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v9
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v0
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[11:12], v[9:10] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[9:10] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[9:10] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v4
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[10:11], v[7:8] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[3:4], v[9:10] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[2:3], v[6:7] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[1:2], v[5:6] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v8
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[0:1], v[13:14] offset1:1
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v4
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[14:15], v[16:17] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v5
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v5, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[14:15], v[4:5] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[14:15] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[6:7] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v0
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[2:3], v[6:7] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[4:5], v[14:15] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[0:1], v[12:13] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v18, v[10:11], v[8:9] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v16i16_to_v16i64:
@@ -7263,236 +7259,235 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 m0, -1
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_mov_b32_e32 v17, v4
-; SI-NEXT: v_mov_b32_e32 v19, v4
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s1
-; SI-NEXT: ds_read2_b64 v[0:3], v12 offset1:1
-; SI-NEXT: ds_read2_b64 v[5:8], v12 offset0:2 offset1:3
-; SI-NEXT: v_mov_b32_e32 v20, s0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_mov_b32_e32 v19, v1
+; SI-NEXT: v_mov_b32_e32 v21, v1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v8
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v8
-; SI-NEXT: ds_read2_b64 v[8:11], v12 offset0:4 offset1:5
-; SI-NEXT: ds_read2_b64 v[12:15], v12 offset0:6 offset1:7
-; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:14 offset1:15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v6
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: ds_read2_b64 v[2:5], v0 offset1:1
+; SI-NEXT: ds_read2_b64 v[6:9], v0 offset0:2 offset1:3
+; SI-NEXT: v_mov_b32_e32 v22, s0
+; SI-NEXT: ds_read2_b64 v[10:13], v0 offset0:4 offset1:5
+; SI-NEXT: ds_read2_b64 v[14:17], v0 offset0:6 offset1:7
+; SI-NEXT: s_waitcnt lgkmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v8
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v8
+; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:12 offset1:13
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v9
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v9
+; SI-NEXT: v_mov_b32_e32 v9, v1
+; SI-NEXT: ds_write2_b64 v22, v[8:9], v[18:19] offset0:14 offset1:15
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v7
+; SI-NEXT: ds_write2_b64 v22, v[18:19], v[8:9] offset0:10 offset1:11
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v6
-; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:10 offset1:11
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v3
-; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:6 offset1:7
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v1
-; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:2 offset1:3
-; SI-NEXT: s_waitcnt lgkmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v15
-; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:30 offset1:31
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v13
-; SI-NEXT: v_mov_b32_e32 v16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v13
-; SI-NEXT: v_mov_b32_e32 v18, v4
-; SI-NEXT: ds_write2_b64 v20, v[17:18], v[15:16] offset0:26 offset1:27
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v11
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v11
-; SI-NEXT: ds_write2_b64 v20, v[17:18], v[15:16] offset0:22 offset1:23
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v7
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_mov_b32_e32 v7, v4
-; SI-NEXT: ds_write2_b64 v20, v[15:16], v[6:7] offset0:12 offset1:13
+; SI-NEXT: v_mov_b32_e32 v8, v1
+; SI-NEXT: ds_write2_b64 v22, v[18:19], v[7:8] offset0:8 offset1:9
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v4
+; SI-NEXT: ds_write2_b64 v22, v[8:9], v[6:7] offset0:4 offset1:5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v5
-; SI-NEXT: ds_write2_b64 v20, v[15:16], v[6:7] offset0:8 offset1:9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v2
-; SI-NEXT: v_mov_b32_e32 v6, v4
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: ds_write2_b64 v20, v[5:6], v[1:2] offset0:4 offset1:5
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v9
-; SI-NEXT: ds_write2_b64 v20, v[6:7], v[1:2] offset1:1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v12
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v14
-; SI-NEXT: ds_write2_b64 v20, v[6:7], v[3:4] offset0:18 offset1:19
-; SI-NEXT: v_mov_b32_e32 v9, v4
-; SI-NEXT: v_mov_b32_e32 v3, v4
-; SI-NEXT: v_mov_b32_e32 v11, v4
-; SI-NEXT: v_mov_b32_e32 v14, v4
-; SI-NEXT: v_mov_b32_e32 v6, v4
-; SI-NEXT: v_mov_b32_e32 v1, v4
-; SI-NEXT: ds_write2_b64 v20, v[17:18], v[10:11] offset0:28 offset1:29
-; SI-NEXT: ds_write2_b64 v20, v[15:16], v[13:14] offset0:24 offset1:25
-; SI-NEXT: ds_write2_b64 v20, v[8:9], v[5:6] offset0:20 offset1:21
-; SI-NEXT: ds_write2_b64 v20, v[2:3], v[0:1] offset0:16 offset1:17
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: ds_write2_b64 v22, v[4:5], v[6:7] offset0:6 offset1:7
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v2
+; SI-NEXT: ds_write2_b64 v22, v[6:7], v[4:5] offset1:1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v3
+; SI-NEXT: v_mov_b32_e32 v3, v1
+; SI-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:2 offset1:3
+; SI-NEXT: s_waitcnt lgkmcnt(8)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v17
+; SI-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:30 offset1:31
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v16
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v15
+; SI-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:28 offset1:29
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_mov_b32_e32 v15, v1
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v13
+; SI-NEXT: ds_write2_b64 v22, v[6:7], v[8:9] offset0:26 offset1:27
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v12
+; SI-NEXT: ds_write2_b64 v22, v[14:15], v[2:3] offset0:24 offset1:25
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v13
+; SI-NEXT: v_mov_b32_e32 v12, v1
+; SI-NEXT: ds_write2_b64 v22, v[11:12], v[4:5] offset0:22 offset1:23
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: ds_write2_b64 v22, v[3:4], v[6:7] offset0:20 offset1:21
+; SI-NEXT: v_mov_b32_e32 v3, v1
+; SI-NEXT: v_mov_b32_e32 v11, v1
+; SI-NEXT: ds_write2_b64 v22, v[2:3], v[0:1] offset0:18 offset1:19
+; SI-NEXT: ds_write2_b64 v22, v[10:11], v[8:9] offset0:16 offset1:17
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v32i16_to_v32i64:
; VI-NO-DS128: ; %bb.0:
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, 0
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v21, v5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, 0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v21, v1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[2:5], v0 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[6:9], v0 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_mov_b32_e32 v22, s0
-; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v4 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v0 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v0 offset1:1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v2
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v5
+; VI-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v5
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:30 offset1:31
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v4
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[18:19] offset0:28 offset1:29
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v3
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[4:5] offset0:26 offset1:27
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v2
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[20:21] offset0:28 offset1:29
-; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v1
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[1:2] offset0:26 offset1:27
-; VI-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[1:2], v[18:19] offset0:24 offset1:25
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v9
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[18:19] offset0:22 offset1:23
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v8
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[8:9] offset0:20 offset1:21
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[7:8] offset0:18 offset1:19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[3:4] offset0:24 offset1:25
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(6)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[2:3] offset0:22 offset1:23
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v8
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[2:3] offset0:20 offset1:21
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[2:3] offset0:18 offset1:19
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v6
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v13
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v12
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v12
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v11
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[5:6], v[2:3] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v1
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v13
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[4:5] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v10
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[5:6], v[7:8] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(10)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v14
-; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v14
-; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v10
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v10
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v17
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v17
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[13:14] offset0:30 offset1:31
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; VI-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v16
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[10:11] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v15
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v15
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[9:10], v[2:3] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[0:1] offset1:1
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[7:8], v[2:3] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v14
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v15
+; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v15
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[10:11], v[4:5] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v14
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v16
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v15, v1
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v17
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v1
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v17
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[14:15], v[6:7] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[10:11], v[2:3] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[8:9] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[0:1] offset0:2 offset1:3
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v32i16_to_v32i64:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v21, v5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v17, v4
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, v4
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v22, s0
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v2
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v2
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[20:21] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v1
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[1:2] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[1:2], v[18:19] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[18:19] offset0:22 offset1:23
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[8:9] offset0:20 offset1:21
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[7:8] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v12 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[5:8], v12 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v20, s0
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v8
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v8
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v12 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[12:15], v12 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v7
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:28 offset1:29
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[16:17] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[16:17], v[6:7] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[16:17], v[5:6] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[2:3], v[5:6] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[5:6], v[2:3] offset0:18 offset1:19
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v4
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v17
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v16
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v14
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[1:2] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v14
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[16:17], v[5:6] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v15
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v14
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v14
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[0:1] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v12
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[2:3] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v12
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v4
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v10
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v12
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v12
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[13:14] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[10:11] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[9:10], v[2:3] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[0:1] offset1:1
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v15, 0xffff, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[1:2], v[5:6] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v10
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v4
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v11
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v8
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v16, v4
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v8
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[9:10], v[0:1] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[11:12], v[7:8] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[5:6], v[13:14] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[15:16], v[3:4] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v32i16_to_v32i64:
@@ -7798,93 +7793,93 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_mov_b32_e32 v1, s1
; VI-DS128-NEXT: ds_read_b128 v[3:6], v1
; VI-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:16
-; VI-DS128-NEXT: v_mov_b32_e32 v52, s0
+; VI-DS128-NEXT: v_mov_b32_e32 v53, s0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v6
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v8
-; VI-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v8
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v10
-; VI-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v10
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v9
-; VI-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v9
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v8
+; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v8
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v7
+; VI-DS128-NEXT: v_and_b32_e32 v21, 0xffff, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v10
+; VI-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v10
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v9
+; VI-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v9
; VI-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:32
-; VI-DS128-NEXT: ds_read_b128 v[29:32], v1 offset:48
-; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v6
+; VI-DS128-NEXT: ds_read_b128 v[30:33], v1 offset:48
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v3
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v4
; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v7
-; VI-DS128-NEXT: v_and_b32_e32 v36, 0xffff, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v9
-; VI-DS128-NEXT: v_and_b32_e32 v42, 0xffff, v9
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v39, 16, v7
+; VI-DS128-NEXT: v_and_b32_e32 v37, 0xffff, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v45, 16, v9
+; VI-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v9
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v30
-; VI-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v30
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v32
-; VI-DS128-NEXT: v_and_b32_e32 v48, 0xffff, v32
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; VI-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v31, 0
-; VI-DS128-NEXT: v_mov_b32_e32 v49, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v51, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v47, 16, v29
-; VI-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v29
-; VI-DS128-NEXT: ds_write_b128 v52, v[48:51] offset:240
-; VI-DS128-NEXT: v_mov_b32_e32 v46, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v48, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v27, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v29, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[45:48] offset:192
-; VI-DS128-NEXT: v_mov_b32_e32 v43, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v45, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[26:29] offset:96
-; VI-DS128-NEXT: v_mov_b32_e32 v24, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v26, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v41, 16, v10
-; VI-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v10
-; VI-DS128-NEXT: ds_write_b128 v52, v[42:45] offset:160
-; VI-DS128-NEXT: v_mov_b32_e32 v40, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v42, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[23:26] offset:112
-; VI-DS128-NEXT: v_mov_b32_e32 v21, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v23, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[39:42] offset:176
-; VI-DS128-NEXT: v_mov_b32_e32 v37, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v39, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[20:23] offset:64
-; VI-DS128-NEXT: v_mov_b32_e32 v18, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v20, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v8
-; VI-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v8
-; VI-DS128-NEXT: v_mov_b32_e32 v8, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v10, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[36:39] offset:128
-; VI-DS128-NEXT: v_mov_b32_e32 v34, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v36, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[17:20] offset:80
-; VI-DS128-NEXT: v_mov_b32_e32 v15, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v17, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; VI-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v3
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v31
+; VI-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v31
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v51, 16, v33
+; VI-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v33
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v32
+; VI-DS128-NEXT: v_and_b32_e32 v31, 0xffff, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v32, 0
+; VI-DS128-NEXT: v_mov_b32_e32 v50, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v52, v32
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v30
+; VI-DS128-NEXT: v_and_b32_e32 v46, 0xffff, v30
+; VI-DS128-NEXT: ds_write_b128 v53, v[49:52] offset:240
+; VI-DS128-NEXT: v_mov_b32_e32 v47, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v49, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[46:49] offset:192
+; VI-DS128-NEXT: v_mov_b32_e32 v44, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v46, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v28, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v30, v32
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v10
+; VI-DS128-NEXT: v_and_b32_e32 v40, 0xffff, v10
+; VI-DS128-NEXT: ds_write_b128 v53, v[43:46] offset:160
+; VI-DS128-NEXT: v_mov_b32_e32 v41, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v43, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[27:30] offset:96
+; VI-DS128-NEXT: v_mov_b32_e32 v25, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v27, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[40:43] offset:176
+; VI-DS128-NEXT: v_mov_b32_e32 v38, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v40, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[24:27] offset:112
+; VI-DS128-NEXT: v_mov_b32_e32 v22, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v24, v32
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v36, 16, v8
+; VI-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v8
+; VI-DS128-NEXT: v_mov_b32_e32 v8, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v10, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[37:40] offset:128
+; VI-DS128-NEXT: v_mov_b32_e32 v35, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v37, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[21:24] offset:64
+; VI-DS128-NEXT: v_mov_b32_e32 v19, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v21, v32
+; VI-DS128-NEXT: v_and_b32_e32 v15, 0xffff, v3
+; VI-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v6
; VI-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; VI-DS128-NEXT: ds_write_b128 v52, v[7:10] offset:208
-; VI-DS128-NEXT: ds_write_b128 v52, v[33:36] offset:144
-; VI-DS128-NEXT: v_mov_b32_e32 v5, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v7, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v33, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[14:17] offset:48
-; VI-DS128-NEXT: v_mov_b32_e32 v12, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v14, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v1, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v3, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[4:7] offset:32
-; VI-DS128-NEXT: ds_write_b128 v52, v[30:33] offset:224
-; VI-DS128-NEXT: ds_write_b128 v52, v[11:14]
-; VI-DS128-NEXT: ds_write_b128 v52, v[0:3] offset:16
+; VI-DS128-NEXT: ds_write_b128 v53, v[7:10] offset:208
+; VI-DS128-NEXT: ds_write_b128 v53, v[34:37] offset:144
+; VI-DS128-NEXT: ds_write_b128 v53, v[18:21] offset:80
+; VI-DS128-NEXT: v_mov_b32_e32 v5, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v7, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v34, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v12, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v14, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v16, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v18, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v1, v32
+; VI-DS128-NEXT: v_mov_b32_e32 v3, v32
+; VI-DS128-NEXT: ds_write_b128 v53, v[4:7] offset:32
+; VI-DS128-NEXT: ds_write_b128 v53, v[31:34] offset:224
+; VI-DS128-NEXT: ds_write_b128 v53, v[11:14] offset:48
+; VI-DS128-NEXT: ds_write_b128 v53, v[15:18]
+; VI-DS128-NEXT: ds_write_b128 v53, v[0:3] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_zextload_v32i16_to_v32i64:
@@ -7909,8 +7904,8 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:32
; GFX9-DS128-NEXT: ds_read_b128 v[29:32], v1 offset:48
; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v6
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v7
; GFX9-DS128-NEXT: v_and_b32_e32 v36, 0xffff, v7
@@ -7963,7 +7958,7 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: ds_write_b128 v52, v[17:20] offset:80
; GFX9-DS128-NEXT: v_mov_b32_e32 v15, v31
; GFX9-DS128-NEXT: v_mov_b32_e32 v17, v31
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v3
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v5
@@ -7994,104 +7989,104 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v5, s1
-; SI-NEXT: ds_read2_b64 v[0:3], v5 offset0:2 offset1:3
-; SI-NEXT: ds_read2_b64 v[6:9], v5 offset1:1
-; SI-NEXT: v_mov_b32_e32 v4, s0
-; SI-NEXT: ds_read2_b64 v[10:13], v5 offset0:6 offset1:7
-; SI-NEXT: ds_read2_b64 v[14:17], v5 offset0:4 offset1:5
+; SI-NEXT: v_mov_b32_e32 v4, s1
+; SI-NEXT: ds_read2_b64 v[0:3], v4 offset0:2 offset1:3
+; SI-NEXT: ds_read2_b64 v[5:8], v4 offset1:1
+; SI-NEXT: v_mov_b32_e32 v21, s0
+; SI-NEXT: ds_read2_b64 v[9:12], v4 offset0:6 offset1:7
+; SI-NEXT: ds_read2_b64 v[13:16], v4 offset0:4 offset1:5
; SI-NEXT: s_waitcnt lgkmcnt(3)
-; SI-NEXT: v_bfe_i32 v20, v3, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v3
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v3
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:14 offset1:15
-; SI-NEXT: v_bfe_i32 v20, v1, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v1
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v1
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:10 offset1:11
+; SI-NEXT: v_bfe_i32 v19, v3, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v3
+; SI-NEXT: v_ashrrev_i32_e32 v17, 16, v3
+; SI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; SI-NEXT: ds_write2_b64 v21, v[19:20], v[17:18] offset0:14 offset1:15
+; SI-NEXT: v_bfe_i32 v17, v1, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v1
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:10 offset1:11
; SI-NEXT: s_waitcnt lgkmcnt(4)
-; SI-NEXT: v_bfe_i32 v20, v9, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v9
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v9
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:6 offset1:7
-; SI-NEXT: v_bfe_i32 v20, v7, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v7
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v7
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:2 offset1:3
+; SI-NEXT: v_bfe_i32 v17, v8, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v8
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v8
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:6 offset1:7
+; SI-NEXT: v_bfe_i32 v17, v6, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v6
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v6
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:2 offset1:3
; SI-NEXT: s_waitcnt lgkmcnt(5)
-; SI-NEXT: v_bfe_i32 v20, v13, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v13
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v13
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:30 offset1:31
-; SI-NEXT: v_bfe_i32 v20, v11, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v11
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v11
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:26 offset1:27
+; SI-NEXT: v_bfe_i32 v17, v12, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v12
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v12
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:30 offset1:31
+; SI-NEXT: v_bfe_i32 v17, v10, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v10
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v10
+; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:26 offset1:27
; SI-NEXT: s_waitcnt lgkmcnt(6)
-; SI-NEXT: v_bfe_i32 v20, v17, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v17
-; SI-NEXT: v_ashrrev_i32_e32 v18, 16, v17
-; SI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; SI-NEXT: ds_write2_b64 v4, v[20:21], v[18:19] offset0:22 offset1:23
-; SI-NEXT: v_bfe_i32 v19, v15, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v15
-; SI-NEXT: v_ashrrev_i32_e32 v17, 16, v15
-; SI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v16
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v16
+; SI-NEXT: v_bfe_i32 v16, v16, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; SI-NEXT: ds_write2_b64 v21, v[16:17], v[3:4] offset0:22 offset1:23
+; SI-NEXT: v_bfe_i32 v16, v14, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v14
+; SI-NEXT: v_ashrrev_i32_e32 v3, 16, v14
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: ds_write2_b64 v4, v[19:20], v[17:18] offset0:18 offset1:19
-; SI-NEXT: v_bfe_i32 v17, v1, 0, 16
+; SI-NEXT: ds_write2_b64 v21, v[16:17], v[3:4] offset0:18 offset1:19
+; SI-NEXT: v_bfe_i32 v3, v1, 0, 16
; SI-NEXT: v_bfe_i32 v1, v2, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; SI-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; SI-NEXT: ds_write2_b64 v4, v[1:2], v[17:18] offset0:12 offset1:13
+; SI-NEXT: ds_write2_b64 v21, v[1:2], v[3:4] offset0:12 offset1:13
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_bfe_i32 v0, v0, 0, 16
; SI-NEXT: v_bfe_i32 v2, v1, 0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v5
; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; SI-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset0:8 offset1:9
-; SI-NEXT: v_bfe_i32 v0, v5, 0, 16
-; SI-NEXT: v_bfe_i32 v2, v8, 0, 16
-; SI-NEXT: v_bfe_i32 v5, v6, 0, 16
-; SI-NEXT: v_bfe_i32 v17, v9, 0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v12
+; SI-NEXT: ds_write2_b64 v21, v[0:1], v[2:3] offset0:8 offset1:9
+; SI-NEXT: v_bfe_i32 v0, v4, 0, 16
+; SI-NEXT: v_bfe_i32 v2, v7, 0, 16
+; SI-NEXT: v_bfe_i32 v4, v5, 0, 16
+; SI-NEXT: v_bfe_i32 v16, v8, 0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; SI-NEXT: ds_write2_b64 v4, v[2:3], v[0:1] offset0:4 offset1:5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v14
-; SI-NEXT: ds_write2_b64 v4, v[5:6], v[17:18] offset1:1
-; SI-NEXT: v_bfe_i32 v5, v11, 0, 16
-; SI-NEXT: v_bfe_i32 v11, v12, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; SI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; SI-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:4 offset1:5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v13
+; SI-NEXT: ds_write2_b64 v21, v[4:5], v[16:17] offset1:1
+; SI-NEXT: v_bfe_i32 v4, v10, 0, 16
+; SI-NEXT: v_bfe_i32 v10, v11, 0, 16
; SI-NEXT: v_bfe_i32 v0, v0, 0, 16
; SI-NEXT: v_bfe_i32 v2, v2, 0, 16
-; SI-NEXT: v_bfe_i32 v7, v7, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; SI-NEXT: v_bfe_i32 v13, v14, 0, 16
-; SI-NEXT: v_bfe_i32 v15, v16, 0, 16
-; SI-NEXT: v_bfe_i32 v9, v10, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; SI-NEXT: v_bfe_i32 v6, v6, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; SI-NEXT: v_bfe_i32 v12, v13, 0, 16
+; SI-NEXT: v_bfe_i32 v14, v15, 0, 16
+; SI-NEXT: v_bfe_i32 v8, v9, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; SI-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; SI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; SI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; SI-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; SI-NEXT: ds_write2_b64 v4, v[11:12], v[5:6] offset0:28 offset1:29
-; SI-NEXT: ds_write2_b64 v4, v[9:10], v[7:8] offset0:24 offset1:25
-; SI-NEXT: ds_write2_b64 v4, v[15:16], v[2:3] offset0:20 offset1:21
-; SI-NEXT: ds_write2_b64 v4, v[13:14], v[0:1] offset0:16 offset1:17
+; SI-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; SI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; SI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; SI-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; SI-NEXT: ds_write2_b64 v21, v[10:11], v[4:5] offset0:28 offset1:29
+; SI-NEXT: ds_write2_b64 v21, v[8:9], v[6:7] offset0:24 offset1:25
+; SI-NEXT: ds_write2_b64 v21, v[14:15], v[2:3] offset0:20 offset1:21
+; SI-NEXT: ds_write2_b64 v21, v[12:13], v[0:1] offset0:16 offset1:17
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_sextload_v32i16_to_v32i64:
@@ -8101,220 +8096,221 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_read2_b64 v[12:15], v4 offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, s0
+; VI-NO-DS128-NEXT: ds_read2_b64 v[5:8], v4 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v21, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v3, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; VI-NO-DS128-NEXT: ds_read2_b64 v[7:10], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_read2_b64 v[3:6], v4 offset1:1
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[16:17], v[18:19] offset0:30 offset1:31
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v2
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v18, v2, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[18:19], v[16:17] offset0:28 offset1:29
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[1:2], v[16:17] offset0:26 offset1:27
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v18, v17, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[16:17], v[18:19] offset0:24 offset1:25
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v18, v15, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[18:19], v[16:17] offset0:22 offset1:23
-; VI-NO-DS128-NEXT: v_bfe_i32 v15, v15, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v14, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[15:16] offset0:20 offset1:21
-; VI-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v13, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-NO-DS128-NEXT: v_bfe_i32 v13, v0, 0, 16
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[16:17], v[14:15] offset0:18 offset1:19
-; VI-NO-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v12
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v12, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v19, v0, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[19:20] offset0:16 offset1:17
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v19, v10, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; VI-NO-DS128-NEXT: v_bfe_i32 v17, v3, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v19, v9, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[19:20], v[17:18] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v9, v9, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[9:10], v[17:18] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_bfe_i32 v9, v0, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v8, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; VI-NO-DS128-NEXT: ds_read2_b64 v[9:12], v4 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_read2_b64 v[13:16], v4 offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[19:20] offset0:30 offset1:31
+; VI-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v17, v2, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[9:10] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v19, v7, 0, 16
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[3:4] offset0:28 offset1:29
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v17, v1, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; VI-NO-DS128-NEXT: v_bfe_i32 v8, v12, 0, 16
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[19:20], v[17:18] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v5, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v5, v6, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[2:3] offset0:26 offset1:27
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v0, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v1, 0, 16
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v8
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[0:1], v[2:3] offset0:24 offset1:25
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v4, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v8, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v7
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:22 offset1:23
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v17, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v7, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v6
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:20 offset1:21
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v18, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v6, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:18 offset1:19
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v19, 0, 16
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v11
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[0:1], v[2:3] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v4, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v11, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v12
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v14
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v7, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v12, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v10
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[4:5], v[2:3] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v10, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-NO-DS128-NEXT: v_bfe_i32 v10, v8, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v8, v9, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v13
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[6:7], v[4:5] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v17, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; VI-NO-DS128-NEXT: v_bfe_i32 v21, v3, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v3, v4, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v22, 31, v21
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; VI-NO-DS128-NEXT: v_bfe_i32 v12, v15, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v20, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v18, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-NO-DS128-NEXT: v_bfe_i32 v17, v14, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[8:9], v[10:11] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_bfe_i32 v8, v13, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v10, v16, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[5:6], v[8:9] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[15:16] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[3:4], v[13:14] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[21:22], v[1:2] offset1:1
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[12:13], v[6:7] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[10:11], v[4:5] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[8:9], v[2:3] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[0:1] offset0:2 offset1:3
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_sextload_v32i16_to_v32i64:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v11 offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v11 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, s0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v8 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v8 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v21, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v8, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[7:10], v11 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[11:14], v11 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v19, v9, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[9:12], v8 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[13:16], v8 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[19:20] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[16:17] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v4
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v17, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[7:8] offset0:28 offset1:29
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v5, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[17:18], v[6:7] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v5, 0, 16
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[3:4], v[16:17] offset0:22 offset1:23
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[4:5], v[6:7] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v8, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[6:7], v[4:5] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v17, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v1
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[3:4] offset0:20 offset1:21
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v1, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[5:6], v[3:4] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v18, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v1, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v0
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[4:5], v[2:3] offset0:18 offset1:19
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v19, 0, 16
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(7)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[0:1], v[2:3] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v15, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v10
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[2:3], v[0:1] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v4, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v16, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v14
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[4:5], v[2:3] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v8, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v14, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v11
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v12
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[6:7], v[4:5] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v18, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v17, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v13, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v10, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v11, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v22, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v20, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[2:3] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v17, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:16 offset1:17
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v14
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v14, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v14, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v13, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[13:14], v[16:17] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v13, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v12, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[13:14] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v11, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v9, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v12, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v9, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v9, v10, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v1, v20, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[16:17], v[14:15] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v8, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 31, v20
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[9:10], v[12:13] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[3:4] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[7:8], v[1:2] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[20:21], v[5:6] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[10:11], v[6:7] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[12:13], v[4:5] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[8:9], v[2:3] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v21, v[18:19], v[0:1] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_sextload_v32i16_to_v32i64:
@@ -8669,15 +8665,15 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: ds_read_b128 v[0:3], v4 offset:48
; VI-DS128-NEXT: ds_read_b128 v[9:12], v4 offset:32
; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
-; VI-DS128-NEXT: ds_read_b128 v[17:20], v4 offset:16
-; VI-DS128-NEXT: ds_read_b128 v[4:7], v4
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; VI-DS128-NEXT: v_bfe_i32 v13, v2, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v15, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-DS128-NEXT: ds_read_b128 v[17:20], v4 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v4
; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:224
; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
@@ -8686,78 +8682,79 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:240
; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; VI-DS128-NEXT: v_bfe_i32 v13, v0, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; VI-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:192
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(5)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v11
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:208
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v11
+; VI-DS128-NEXT: v_bfe_i32 v2, v13, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:160
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v12
+; VI-DS128-NEXT: v_bfe_i32 v2, v14, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v12, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:176
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; VI-DS128-NEXT: v_bfe_i32 v2, v15, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v9, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v10
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:128
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; VI-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; VI-DS128-NEXT: v_bfe_i32 v9, v10, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v19
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:144
-; VI-DS128-NEXT: v_bfe_i32 v9, v19, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v20
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:96
-; VI-DS128-NEXT: v_bfe_i32 v9, v20, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v17
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:112
-; VI-DS128-NEXT: v_bfe_i32 v9, v17, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:64
-; VI-DS128-NEXT: v_bfe_i32 v9, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: v_bfe_i32 v11, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v18
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:192
+; VI-DS128-NEXT: v_bfe_i32 v0, v10, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v16, 0, 16
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(8)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v19
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:144
+; VI-DS128-NEXT: v_bfe_i32 v2, v11, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v19, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v20
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:96
+; VI-DS128-NEXT: v_bfe_i32 v0, v20, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v12, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v17
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:112
+; VI-DS128-NEXT: v_bfe_i32 v0, v17, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v9, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v18
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(10)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v6
+; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:64
+; VI-DS128-NEXT: v_bfe_i32 v2, v13, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v15, v15, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; VI-DS128-NEXT: v_bfe_i32 v9, v4, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v6, v21, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:80
; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v15, v22, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
@@ -8780,15 +8777,15 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v13 offset:48
; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v13 offset:32
; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v13
-; GFX9-DS128-NEXT: ds_read_b128 v[18:21], v13 offset:16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v6
; GFX9-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v8, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v13
+; GFX9-DS128-NEXT: ds_read_b128 v[18:21], v13 offset:16
; GFX9-DS128-NEXT: ds_write_b128 v12, v[14:17] offset:224
; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
@@ -8797,91 +8794,90 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:240
; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; GFX9-DS128-NEXT: v_bfe_i32 v13, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-DS128-NEXT: v_bfe_i32 v4, v5, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:192
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(5)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:208
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(5)
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v13, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v4, v2, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:160
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v3, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v14, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:176
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v15, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v2, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:176
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v1
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:128
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:192
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v1, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v16, 0, 16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v20
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:144
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v20, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v21
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v21, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:112
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:64
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v20
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[0:3] offset:144
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v20, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[0:3] offset:96
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v21, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v7, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v18
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[0:3] offset:112
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v18, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v10
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[0:3] offset:64
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v13, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v13, v19, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v15, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v11
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v8
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v8, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v10, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v17, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v9, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v5, 0, 16
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:80
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-DS128-NEXT: v_bfe_i32 v17, v10, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v19, v0, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v9, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v22, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[17:20] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[8:11] offset:32
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[6:9]
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7]
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[0:3] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = sext <32 x i16> %load to <32 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
index d7657fafca24e..1ffc3a06a9239 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
@@ -35,19 +35,24 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
; GFX942-SDAG-LABEL: buffer_nontemporal_load_store:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942-SDAG-NEXT: s_load_dword s11, s[4:5], 0x10
+; GFX942-SDAG-NEXT: s_load_dword s7, s[4:5], 0x10
+; GFX942-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX942-SDAG-NEXT: s_mov_b32 s11, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s10, s3
; GFX942-SDAG-NEXT: s_mov_b32 s8, s1
+; GFX942-SDAG-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
; GFX942-SDAG-NEXT: s_mov_b32 s9, s2
-; GFX942-SDAG-NEXT: s_mov_b32 s10, s3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX942-SDAG-NEXT: buffer_load_dword v1, v0, s[8:11], 0 offen nt
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20
; GFX942-SDAG-NEXT: s_load_dword s7, s[4:5], 0x30
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20
+; GFX942-SDAG-NEXT: s_mov_b32 s5, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s4, s3
+; GFX942-SDAG-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
; GFX942-SDAG-NEXT: s_mov_b32 s4, s1
; GFX942-SDAG-NEXT: s_mov_b32 s5, s2
-; GFX942-SDAG-NEXT: s_mov_b32 s6, s3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: buffer_store_dword v1, v0, s[4:7], 0 offen nt
@@ -175,20 +180,25 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
; GFX942-SDAG-LABEL: buffer_nontemporal_and_volatile_load_store:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942-SDAG-NEXT: s_load_dword s11, s[4:5], 0x10
+; GFX942-SDAG-NEXT: s_load_dword s7, s[4:5], 0x10
+; GFX942-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX942-SDAG-NEXT: s_mov_b32 s11, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s10, s3
; GFX942-SDAG-NEXT: s_mov_b32 s8, s1
+; GFX942-SDAG-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
; GFX942-SDAG-NEXT: s_mov_b32 s9, s2
-; GFX942-SDAG-NEXT: s_mov_b32 s10, s3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX942-SDAG-NEXT: buffer_load_dword v1, v0, s[8:11], 0 offen sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20
; GFX942-SDAG-NEXT: s_load_dword s7, s[4:5], 0x30
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20
+; GFX942-SDAG-NEXT: s_mov_b32 s5, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s4, s3
+; GFX942-SDAG-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
; GFX942-SDAG-NEXT: s_mov_b32 s4, s1
; GFX942-SDAG-NEXT: s_mov_b32 s5, s2
-; GFX942-SDAG-NEXT: s_mov_b32 s6, s3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX942-SDAG-NEXT: buffer_store_dword v1, v0, s[4:7], 0 offen sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index 61f79ccd55f1c..3a126ea7f0d90 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -185,22 +185,22 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:328
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:332
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328
-; ALIGNED-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:332
-; ALIGNED-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:320
-; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:320
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v39 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v39 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v34 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v34 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v33 offset:28
-; ALIGNED-NEXT: flat_store_byte v[24:25], v33 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v33 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:28
+; ALIGNED-NEXT: flat_store_byte v[24:25], v32 offset:32
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:272
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:276
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:280
@@ -509,7 +509,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 8, v32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 24, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v67
-; ALIGNED-NEXT: flat_store_byte v[20:21], v101 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v101 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v54
; ALIGNED-NEXT: v_lshrrev_b32_e32 v54, 8, v54
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v66
@@ -525,25 +525,25 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v81
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 24, v97
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 8, v97
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v87
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v87
-; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v39, 24, v83
; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v83
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v68
; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 8, v68
-; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v34, 24, v96
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 8, v96
-; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v84
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 8, v84
-; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v33, 24, v69
; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v69
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 24, v85
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v85
; ALIGNED-NEXT: flat_store_byte v[20:21], v102 offset:28
@@ -766,17 +766,17 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; UNROLL3-NEXT: v_add_co_u32 v16, vcc_lo, v0, s4
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: flat_load_dwordx4 v[4:7], v[12:13]
-; UNROLL3-NEXT: flat_load_dwordx4 v[8:11], v[12:13] offset:16
-; UNROLL3-NEXT: flat_load_dwordx4 v[12:15], v[12:13] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[4:7], v[12:13] offset:16
+; UNROLL3-NEXT: flat_load_dwordx4 v[8:11], v[12:13] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v17, null, s5, v1, vcc_lo
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: s_waitcnt vmcnt(2) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
+; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[4:7] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[8:11] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[12:15]
; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB0_1
@@ -1192,22 +1192,22 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:280
; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:280
-; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:284
-; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:272
-; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284
+; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:280
+; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:276
+; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:272
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v15, off offset:42
-; ALIGNED-NEXT: global_store_byte v[16:17], v15, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v15, off offset:46
+; ALIGNED-NEXT: global_store_byte v[16:17], v15, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v14, off offset:46
-; ALIGNED-NEXT: global_store_byte v[16:17], v14, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v14, off offset:42
+; ALIGNED-NEXT: global_store_byte v[16:17], v14, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v13, off offset:34
-; ALIGNED-NEXT: global_store_byte v[16:17], v13, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v13, off offset:38
+; ALIGNED-NEXT: global_store_byte v[16:17], v13, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v12, off offset:38
-; ALIGNED-NEXT: global_store_byte v[16:17], v12, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[16:17], v12, off offset:34
+; ALIGNED-NEXT: global_store_byte v[16:17], v12, off offset:32
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:228
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:232
@@ -1450,14 +1450,14 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: global_store_byte v[16:17], v20, off offset:53
; ALIGNED-NEXT: global_store_byte v[16:17], v82, off offset:51
; ALIGNED-NEXT: global_store_byte v[16:17], v18, off offset:49
-; ALIGNED-NEXT: global_store_byte v[16:17], v71, off offset:43
-; ALIGNED-NEXT: global_store_byte v[16:17], v15, off offset:41
-; ALIGNED-NEXT: global_store_byte v[16:17], v66, off offset:47
-; ALIGNED-NEXT: global_store_byte v[16:17], v14, off offset:45
-; ALIGNED-NEXT: global_store_byte v[16:17], v30, off offset:35
-; ALIGNED-NEXT: global_store_byte v[16:17], v13, off offset:33
-; ALIGNED-NEXT: global_store_byte v[16:17], v98, off offset:39
-; ALIGNED-NEXT: global_store_byte v[16:17], v12, off offset:37
+; ALIGNED-NEXT: global_store_byte v[16:17], v71, off offset:47
+; ALIGNED-NEXT: global_store_byte v[16:17], v15, off offset:45
+; ALIGNED-NEXT: global_store_byte v[16:17], v66, off offset:43
+; ALIGNED-NEXT: global_store_byte v[16:17], v14, off offset:41
+; ALIGNED-NEXT: global_store_byte v[16:17], v30, off offset:39
+; ALIGNED-NEXT: global_store_byte v[16:17], v13, off offset:37
+; ALIGNED-NEXT: global_store_byte v[16:17], v98, off offset:35
+; ALIGNED-NEXT: global_store_byte v[16:17], v12, off offset:33
; ALIGNED-NEXT: global_store_byte v[16:17], v70, off offset:31
; ALIGNED-NEXT: global_store_byte v[16:17], v11, off offset:29
; ALIGNED-NEXT: global_store_byte v[16:17], v83, off offset:27
@@ -1520,17 +1520,17 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; UNROLL3-NEXT: v_add_co_u32 v16, vcc_lo, v0, s4
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: global_load_dwordx4 v[4:7], v[12:13], off
-; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[12:13], off offset:16
-; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[4:7], v[12:13], off offset:16
+; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[12:13], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[12:13], off
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v17, null, s5, v1, vcc_lo
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
-; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[4:7], off
+; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[4:7], off offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[8:11], off offset:16
+; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[8:11], off offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[12:15], off offset:32
+; UNROLL3-NEXT: global_store_dwordx4 v[16:17], v[12:15], off
; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB1_1
@@ -1776,13 +1776,13 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: flat_store_byte v[86:87], v112 offset:198
; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v55
; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:204
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v53
; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:202
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v53
; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:196
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v52
; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:194
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v52
; ALIGNED-NEXT: flat_store_byte v[86:87], v114 offset:192
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 24, v50
@@ -1814,18 +1814,18 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: flat_store_byte v[86:87], v103 offset:178
; ALIGNED-NEXT: flat_store_byte v[86:87], v68 offset:176
; ALIGNED-NEXT: flat_store_byte v[86:87], v69 offset:174
-; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v54 offset:164
-; ALIGNED-NEXT: flat_store_byte v[86:87], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v55 offset:168
; ALIGNED-NEXT: flat_store_byte v[86:87], v55 offset:169
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
-; ALIGNED-NEXT: flat_store_byte v[86:87], v52 offset:157
+; ALIGNED-NEXT: flat_store_byte v[86:87], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v53 offset:160
; ALIGNED-NEXT: flat_store_byte v[86:87], v53 offset:161
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
+; ALIGNED-NEXT: flat_store_byte v[86:87], v52 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v38
@@ -1846,12 +1846,12 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v33
; ALIGNED-NEXT: flat_store_byte v[86:87], v112 offset:170
; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v33
-; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:160
+; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:164
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v32
-; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:158
+; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:162
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v32
-; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:164
-; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:162
+; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:160
+; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:158
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:200
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:204
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:196
@@ -2133,16 +2133,16 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_clause 0x2
; UNROLL3-NEXT: global_load_dwordx4 v[4:7], v[12:13], off offset:16
-; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[12:13], off
-; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[12:13], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[12:13], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[12:13], off
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v17, null, s5, v1, vcc_lo
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[4:7] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
+; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[8:11] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[12:15] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[12:15]
; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB2_1
@@ -3777,50 +3777,50 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: .LBB4_1: ; %static-memcpy-expansion-main-body
; ALIGNED-NEXT: ; =>This Inner Loop Header: Depth=1
; ALIGNED-NEXT: s_clause 0x3b
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:20
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:22
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:20
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:21
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:23
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:24
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:25
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:25
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:26
; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:27
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:29
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:30
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:31
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:32
; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:35
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:37
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:38
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:39
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:40
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:41
; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:42
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:43
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:44
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:45
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:46
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:47
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:48
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:49
-; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:50
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:51
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:51
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:52
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:53
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:54
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:55
; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:56
; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:57
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:58
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:60
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:61
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:62
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:64
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:65
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:66
@@ -3838,77 +3838,75 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:77
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:78
; ALIGNED-NEXT: s_waitcnt vmcnt(59)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(58)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(58)
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(57)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(56)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(56)
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(55)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(54)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(53)
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v1
; ALIGNED-NEXT: s_waitcnt vmcnt(51)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(50)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(49)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(48)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(47)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v5
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v8, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 8, v11
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v3, 8, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
; ALIGNED-NEXT: s_waitcnt vmcnt(45)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v15, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v15, 8, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(43)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v20, 8, v19
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v17, 8, v16
; ALIGNED-NEXT: s_waitcnt vmcnt(41)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v13
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v14, 8, v18
; ALIGNED-NEXT: s_waitcnt vmcnt(39)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
-; ALIGNED-NEXT: s_waitcnt vmcnt(37)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v27, 8, v22
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v19, 8, v20
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: s_waitcnt vmcnt(35)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v29, 8, v28
; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 16, v5
+; ALIGNED-NEXT: s_waitcnt vmcnt(34)
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v21, 8, v22
; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 16, v7
+; ALIGNED-NEXT: s_waitcnt vmcnt(32)
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v25, 8, v24
; ALIGNED-NEXT: v_lshl_or_b32 v5, v10, 16, v9
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 16, v11
-; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(33)
-; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
-; ALIGNED-NEXT: s_waitcnt vmcnt(31)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
+; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v13, v27, 8, v26
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 16, v11
+; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v29, 8, v28
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(29)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v31, 8, v30
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(27)
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v34, 8, v33
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v35, 8, v34
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v32
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v33, 8, v32
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(23)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v35
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v37, 8, v36
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(18)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v49, 8, v21
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v39, 8, v23
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(16)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v48, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v49, 8, v48
; ALIGNED-NEXT: v_lshl_or_b32 v7, v14, 16, v13
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
@@ -3916,7 +3914,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 16, v5
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v38
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v52, 8, v51
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
@@ -3939,63 +3937,63 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v1, v68, 8, v67
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v82, 8, v81
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v65, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v65, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v70, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:91
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:112
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:113
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:115
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:114
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:115
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
@@ -4003,11 +4001,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v83
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v80, 8, v71
@@ -4016,31 +4014,31 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v3, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:86
; ALIGNED-NEXT: v_lshl_or_b32 v1, v6, 8, v5
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:87
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:88
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:89
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:85
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:84
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:84
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:85
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(4)
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
@@ -4049,24 +4047,24 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 8, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v1
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v3, 8, v4
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:94
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:95
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:93
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:92
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:93
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 8, v0
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v1
; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v3
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:99
@@ -4088,7 +4086,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
@@ -4102,202 +4100,202 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 8, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v3, 8, v5
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:103
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:106
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:103
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:101
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v5, 8, v1
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:110
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:111
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:113
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:111
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:109
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:119
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:118
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v9
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:129
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v8, 8, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v8, 8, v5
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:122
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:124
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v7, 8, v6
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:120
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:121
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:123
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v13, 8, v12
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:130
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:131
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v11, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 8, v10
; ALIGNED-NEXT: s_clause 0x1
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:122
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:128
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:129
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:118
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:119
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:117
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:127
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:126
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v5
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v11
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:132
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:126
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:127
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:125
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v8
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v8, 8, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v7, 8, v4
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 8, v5
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v13, 8, v12
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v11, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v9, 8, v10
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:133
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:134
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:135
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:136
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:137
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:138
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:139
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:134
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:135
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:136
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:137
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:140
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:144
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v11
; ALIGNED-NEXT: s_clause 0x3e
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:165
; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:167
; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:168
-; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:169
; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:170
; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:173
-; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:174
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:175
; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:176
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:177
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:178
; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:180
-; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:181
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:182
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:183
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:184
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:185
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:186
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:190
; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:191
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:192
-; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:193
; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:194
; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:195
-; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:196
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:197
-; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:198
; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:199
-; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:200
; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:201
; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:202
; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:204
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:205
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:206
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:207
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:208
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:209
-; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:210
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:212
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:213
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:214
-; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:215
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:216
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:217
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:218
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:219
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:220
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:221
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:222
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:223
; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:224
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:225
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:226
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:227
; ALIGNED-NEXT: s_clause 0x24
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:228
-; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:229
-; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:230
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:231
; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:232
; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:233
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:234
; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:235
; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:236
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:237
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:238
; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:239
; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:240
; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:241
@@ -4305,194 +4303,194 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:243
; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:244
; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen
-; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:1
; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:2
-; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:3
-; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:4
-; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:5
-; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:6
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:7
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:12
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:13
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:14
; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:15
; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:11
; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:8
; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:9
; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:10
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:145
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:146
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:147
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:148
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:143
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:142
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:142
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:143
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:141
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(19)
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v120, 8, v121
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v122, 8, v121
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:150
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v9
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v8
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:156
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v8, 8, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v7, 8, v4
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:154
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:155
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 8, v5
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:152
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:153
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:153
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v13, 8, v12
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:162
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:163
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v11, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v9, 8, v10
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:160
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:161
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:150
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:151
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:149
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:158
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v5
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v11
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:164
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:159
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 8, v1
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v84
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v84, 8, v85
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v8
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v8, 8, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v7, 8, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v83
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v7, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v61, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v97, 8, v99
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v73, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v97, 8, v98
; ALIGNED-NEXT: v_lshl_or_b32 v4, v96, 8, v87
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v13, 8, v12
; ALIGNED-NEXT: v_lshl_or_b32 v59, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v81
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v11, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v9, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v69
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v108, 8, v109
+; ALIGNED-NEXT: v_lshl_or_b32 v41, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v80
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v65
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v110, 8, v11
; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:245
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:246
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:247
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:248
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:249
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:250
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:251
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v64
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v108, 8, v109
-; ALIGNED-NEXT: v_lshl_or_b32 v116, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v65, 8, v67
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v54, 8, v55
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v111, 8, v5
; ALIGNED-NEXT: v_lshl_or_b32 v103, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v66
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v50, 8, v51
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v79, 8, v88
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v50
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v88, 8, v79
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v49
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v89, 8, v93
-; ALIGNED-NEXT: v_lshl_or_b32 v98, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v80
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v89, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v99, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v55, 8, v64
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v53
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v104, 8, v95
-; ALIGNED-NEXT: v_lshl_or_b32 v85, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v36, 8, v39
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v105, 8, v107
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v39
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v106, 8, v107
; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v35
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v62, 8, v63
-; ALIGNED-NEXT: v_lshl_or_b32 v69, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v37
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v32
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v62, 8, v61
+; ALIGNED-NEXT: v_lshl_or_b32 v68, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v32, 8, v36
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v31
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v72, 8, v75
-; ALIGNED-NEXT: v_lshl_or_b32 v51, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v23, 8, v25
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v63, 8, v75
+; ALIGNED-NEXT: v_lshl_or_b32 v54, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v24, 8, v25
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v46, 8, v58
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v24
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v46, 8, v47
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v23
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v47, 8, v73
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v58, 8, v72
; ALIGNED-NEXT: v_lshl_or_b32 v38, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v28, 8, v29
; ALIGNED-NEXT: v_lshl_or_b32 v4, v26, 8, v27
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v118, 8, v40
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v118, 8, v117
; ALIGNED-NEXT: v_lshl_or_b32 v33, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v21
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v119, 8, v41
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v119, 8, v40
; ALIGNED-NEXT: v_lshl_or_b32 v4, v18, 8, v17
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v42, 8, v43
@@ -4502,38 +4500,38 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v112, 8, v113
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v114, 8, v117
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v115, 8, v116
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v100, 8, v102
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v100, 8, v101
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v101, 8, v115
-; ALIGNED-NEXT: v_lshl_or_b32 v106, v1, 16, v0
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v102, 8, v114
+; ALIGNED-NEXT: v_lshl_or_b32 v105, v1, 16, v0
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:253
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:254
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:255
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v15, 8, v20
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v16, 8, v20
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v14, 8, v16
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v14, 8, v15
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: v_lshl_or_b32 v56, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 16, v4
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v7, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v8, 8, v9
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v1, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v1, 8, v7
; ALIGNED-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 8, v13
; ALIGNED-NEXT: v_lshl_or_b32 v6, v56, 16, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v56, v124, 8, v126
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v125, 8, v126
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
-; ALIGNED-NEXT: v_lshl_or_b32 v56, v123, 8, v125
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v110, 8, v122
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v124, 8, v123
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v120, 8, v111
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
-; ALIGNED-NEXT: v_lshl_or_b32 v56, v91, 8, v94
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v90, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v94, 8, v93
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v90, 8, v91
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
; ALIGNED-NEXT: v_lshl_or_b32 v56, v76, 8, v78
@@ -4545,7 +4543,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:16
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:17
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
@@ -4568,62 +4566,62 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v11 offset:247
; ALIGNED-NEXT: flat_store_byte v[5:6], v10 offset:248
; ALIGNED-NEXT: flat_store_byte v[5:6], v12 offset:246
+; ALIGNED-NEXT: flat_store_byte v[5:6], v7 offset:251
; ALIGNED-NEXT: flat_store_byte v[5:6], v1 offset:252
-; ALIGNED-NEXT: flat_store_byte v[5:6], v7 offset:250
-; ALIGNED-NEXT: flat_store_byte v[5:6], v8 offset:251
; ALIGNED-NEXT: flat_store_byte v[5:6], v9 offset:249
+; ALIGNED-NEXT: flat_store_byte v[5:6], v8 offset:250
; ALIGNED-NEXT: flat_store_byte v[5:6], v13 offset:245
; ALIGNED-NEXT: flat_store_byte v[5:6], v17 offset:239
; ALIGNED-NEXT: flat_store_byte v[5:6], v18 offset:240
; ALIGNED-NEXT: flat_store_byte v[5:6], v19 offset:238
+; ALIGNED-NEXT: flat_store_byte v[5:6], v15 offset:243
; ALIGNED-NEXT: flat_store_byte v[5:6], v14 offset:244
-; ALIGNED-NEXT: flat_store_byte v[5:6], v15 offset:242
-; ALIGNED-NEXT: flat_store_byte v[5:6], v16 offset:243
; ALIGNED-NEXT: flat_store_byte v[5:6], v20 offset:241
+; ALIGNED-NEXT: flat_store_byte v[5:6], v16 offset:242
; ALIGNED-NEXT: flat_store_byte v[5:6], v21 offset:237
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:248
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte v[5:6], v27 offset:231
; ALIGNED-NEXT: flat_store_byte v[5:6], v26 offset:232
; ALIGNED-NEXT: flat_store_byte v[5:6], v28 offset:230
+; ALIGNED-NEXT: flat_store_byte v[5:6], v23 offset:235
; ALIGNED-NEXT: flat_store_byte v[5:6], v22 offset:236
-; ALIGNED-NEXT: flat_store_byte v[5:6], v23 offset:234
-; ALIGNED-NEXT: flat_store_byte v[5:6], v24 offset:235
; ALIGNED-NEXT: flat_store_byte v[5:6], v25 offset:233
+; ALIGNED-NEXT: flat_store_byte v[5:6], v24 offset:234
; ALIGNED-NEXT: flat_store_byte v[5:6], v29 offset:229
; ALIGNED-NEXT: flat_store_byte v[5:6], v35 offset:223
; ALIGNED-NEXT: flat_store_byte v[5:6], v34 offset:224
-; ALIGNED-NEXT: flat_store_byte v[5:6], v36 offset:222
+; ALIGNED-NEXT: flat_store_byte v[5:6], v37 offset:222
+; ALIGNED-NEXT: flat_store_byte v[5:6], v31 offset:227
; ALIGNED-NEXT: flat_store_byte v[5:6], v30 offset:228
-; ALIGNED-NEXT: flat_store_byte v[5:6], v31 offset:226
-; ALIGNED-NEXT: flat_store_byte v[5:6], v32 offset:227
-; ALIGNED-NEXT: flat_store_byte v[5:6], v37 offset:225
+; ALIGNED-NEXT: flat_store_byte v[5:6], v36 offset:225
+; ALIGNED-NEXT: flat_store_byte v[5:6], v32 offset:226
; ALIGNED-NEXT: flat_store_byte v[5:6], v39 offset:221
-; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:192
-; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: flat_store_byte v[5:6], v53 offset:210
-; ALIGNED-NEXT: flat_store_byte v[5:6], v52 offset:212
-; ALIGNED-NEXT: flat_store_byte v[5:6], v71 offset:206
-; ALIGNED-NEXT: flat_store_byte v[5:6], v70 offset:208
-; ALIGNED-NEXT: flat_store_byte v[5:6], v80 offset:207
-; ALIGNED-NEXT: flat_store_byte v[5:6], v64 offset:211
-; ALIGNED-NEXT: flat_store_byte v[5:6], v68 offset:209
-; ALIGNED-NEXT: flat_store_byte v[5:6], v55 offset:215
-; ALIGNED-NEXT: flat_store_byte v[5:6], v54 offset:216
-; ALIGNED-NEXT: flat_store_byte v[5:6], v65 offset:214
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: flat_store_byte v[5:6], v53 offset:215
+; ALIGNED-NEXT: flat_store_byte v[5:6], v52 offset:216
+; ALIGNED-NEXT: flat_store_byte v[5:6], v55 offset:214
+; ALIGNED-NEXT: flat_store_byte v[5:6], v49 offset:219
; ALIGNED-NEXT: flat_store_byte v[5:6], v48 offset:220
-; ALIGNED-NEXT: flat_store_byte v[5:6], v49 offset:218
-; ALIGNED-NEXT: flat_store_byte v[5:6], v50 offset:219
-; ALIGNED-NEXT: flat_store_byte v[5:6], v66 offset:217
-; ALIGNED-NEXT: flat_store_byte v[5:6], v67 offset:213
+; ALIGNED-NEXT: flat_store_byte v[5:6], v51 offset:217
+; ALIGNED-NEXT: flat_store_byte v[5:6], v50 offset:218
+; ALIGNED-NEXT: flat_store_byte v[5:6], v64 offset:213
+; ALIGNED-NEXT: flat_store_byte v[5:6], v69 offset:207
+; ALIGNED-NEXT: flat_store_byte v[5:6], v70 offset:208
+; ALIGNED-NEXT: flat_store_byte v[5:6], v71 offset:206
+; ALIGNED-NEXT: flat_store_byte v[5:6], v65 offset:211
+; ALIGNED-NEXT: flat_store_byte v[5:6], v66 offset:212
+; ALIGNED-NEXT: flat_store_byte v[5:6], v80 offset:209
+; ALIGNED-NEXT: flat_store_byte v[5:6], v67 offset:210
; ALIGNED-NEXT: flat_store_byte v[5:6], v81 offset:205
; ALIGNED-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v106, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v73, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v105, off, s[0:3], s32 offset:212
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
; ALIGNED-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; ALIGNED-NEXT: s_and_b32 vcc_lo, exec_lo, s6
@@ -4633,19 +4631,19 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v87 offset:199
; ALIGNED-NEXT: flat_store_byte v[5:6], v96 offset:200
; ALIGNED-NEXT: flat_store_byte v[5:6], v97 offset:198
+; ALIGNED-NEXT: flat_store_byte v[5:6], v83 offset:203
; ALIGNED-NEXT: flat_store_byte v[5:6], v82 offset:204
-; ALIGNED-NEXT: flat_store_byte v[5:6], v83 offset:202
-; ALIGNED-NEXT: flat_store_byte v[5:6], v84 offset:203
-; ALIGNED-NEXT: flat_store_byte v[5:6], v86 offset:201
-; ALIGNED-NEXT: flat_store_byte v[5:6], v99 offset:197
+; ALIGNED-NEXT: flat_store_byte v[5:6], v85 offset:201
+; ALIGNED-NEXT: flat_store_byte v[5:6], v84 offset:202
+; ALIGNED-NEXT: flat_store_byte v[5:6], v98 offset:197
; ALIGNED-NEXT: flat_store_byte v[5:6], v113 offset:191
; ALIGNED-NEXT: flat_store_byte v[5:6], v112 offset:192
-; ALIGNED-NEXT: flat_store_byte v[5:6], v114 offset:190
+; ALIGNED-NEXT: flat_store_byte v[5:6], v115 offset:190
+; ALIGNED-NEXT: flat_store_byte v[5:6], v101 offset:195
; ALIGNED-NEXT: flat_store_byte v[5:6], v100 offset:196
-; ALIGNED-NEXT: flat_store_byte v[5:6], v101 offset:194
-; ALIGNED-NEXT: flat_store_byte v[5:6], v102 offset:195
-; ALIGNED-NEXT: flat_store_byte v[5:6], v115 offset:193
-; ALIGNED-NEXT: flat_store_byte v[5:6], v117 offset:189
+; ALIGNED-NEXT: flat_store_byte v[5:6], v114 offset:193
+; ALIGNED-NEXT: flat_store_byte v[5:6], v102 offset:194
+; ALIGNED-NEXT: flat_store_byte v[5:6], v116 offset:189
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
@@ -4661,22 +4659,22 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v43 offset:183
; ALIGNED-NEXT: flat_store_byte v[5:6], v42 offset:184
; ALIGNED-NEXT: flat_store_byte v[5:6], v44 offset:182
+; ALIGNED-NEXT: flat_store_byte v[5:6], v117 offset:187
; ALIGNED-NEXT: flat_store_byte v[5:6], v118 offset:188
+; ALIGNED-NEXT: flat_store_byte v[5:6], v40 offset:185
; ALIGNED-NEXT: flat_store_byte v[5:6], v119 offset:186
-; ALIGNED-NEXT: flat_store_byte v[5:6], v40 offset:187
-; ALIGNED-NEXT: flat_store_byte v[5:6], v41 offset:185
; ALIGNED-NEXT: flat_store_byte v[5:6], v45 offset:181
-; ALIGNED-NEXT: flat_store_byte v[5:6], v63 offset:175
+; ALIGNED-NEXT: flat_store_byte v[5:6], v61 offset:175
; ALIGNED-NEXT: flat_store_byte v[5:6], v62 offset:176
-; ALIGNED-NEXT: flat_store_byte v[5:6], v72 offset:174
+; ALIGNED-NEXT: flat_store_byte v[5:6], v63 offset:174
+; ALIGNED-NEXT: flat_store_byte v[5:6], v47 offset:179
; ALIGNED-NEXT: flat_store_byte v[5:6], v46 offset:180
-; ALIGNED-NEXT: flat_store_byte v[5:6], v47 offset:178
-; ALIGNED-NEXT: flat_store_byte v[5:6], v58 offset:179
-; ALIGNED-NEXT: flat_store_byte v[5:6], v73 offset:177
+; ALIGNED-NEXT: flat_store_byte v[5:6], v72 offset:177
+; ALIGNED-NEXT: flat_store_byte v[5:6], v58 offset:178
; ALIGNED-NEXT: flat_store_byte v[5:6], v75 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
@@ -4688,11 +4686,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[5:6], v95 offset:167
; ALIGNED-NEXT: flat_store_byte v[5:6], v104 offset:168
-; ALIGNED-NEXT: flat_store_byte v[5:6], v105 offset:166
-; ALIGNED-NEXT: flat_store_byte v[5:6], v79 offset:172
+; ALIGNED-NEXT: flat_store_byte v[5:6], v106 offset:166
+; ALIGNED-NEXT: flat_store_byte v[5:6], v79 offset:171
+; ALIGNED-NEXT: flat_store_byte v[5:6], v88 offset:172
+; ALIGNED-NEXT: flat_store_byte v[5:6], v92 offset:169
; ALIGNED-NEXT: flat_store_byte v[5:6], v89 offset:170
-; ALIGNED-NEXT: flat_store_byte v[5:6], v88 offset:171
-; ALIGNED-NEXT: flat_store_byte v[5:6], v93 offset:169
; ALIGNED-NEXT: flat_store_byte v[5:6], v107 offset:165
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:159
@@ -4702,12 +4700,12 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:158
-; ALIGNED-NEXT: flat_store_byte v[5:6], v108 offset:164
-; ALIGNED-NEXT: flat_store_byte v[5:6], v111 offset:162
; ALIGNED-NEXT: flat_store_byte v[5:6], v109 offset:163
+; ALIGNED-NEXT: flat_store_byte v[5:6], v108 offset:164
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:161
+; ALIGNED-NEXT: flat_store_byte v[5:6], v110 offset:162
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:157
@@ -4734,16 +4732,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:150
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:155
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:156
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:154
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:155
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:153
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:153
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:154
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:149
@@ -4758,16 +4756,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:142
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:147
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:148
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:146
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:147
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:145
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:145
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:146
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:141
@@ -4794,16 +4792,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:134
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:139
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:140
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:138
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:139
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:137
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:137
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:138
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:133
@@ -4818,16 +4816,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:126
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:131
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:132
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:130
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:131
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:129
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:130
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:128
@@ -4854,16 +4852,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:118
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:123
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:124
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:122
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:123
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:121
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:121
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:122
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:117
@@ -4878,16 +4876,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:110
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:115
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:116
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:114
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:115
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:113
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:113
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:114
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:109
@@ -4914,16 +4912,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:102
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:107
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:108
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:106
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:107
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:105
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:105
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:106
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:101
@@ -4938,16 +4936,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:94
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:99
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:100
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:98
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:99
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:97
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:97
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:98
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:93
@@ -4957,7 +4955,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
@@ -4974,16 +4972,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:86
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:91
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:92
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:90
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:91
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:89
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:89
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:90
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:85
@@ -4998,16 +4996,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:78
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:83
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:84
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:82
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:81
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:83
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:81
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:82
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:77
@@ -5034,16 +5032,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:70
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:75
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:76
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:74
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:75
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:73
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:73
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:74
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:69
@@ -5058,16 +5056,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:62
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:67
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:68
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:66
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:67
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:65
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:65
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:66
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:64
@@ -5080,13 +5078,10 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
@@ -5095,48 +5090,51 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:54
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:59
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:60
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:59
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:57
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:57
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:58
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:46
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:52
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:51
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:51
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:52
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:49
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:50
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436
@@ -5145,52 +5143,52 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:40
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:39
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:39
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:37
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:44
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:43
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:43
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:44
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:42
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:41
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:41
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:42
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:37
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:32
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:31
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:31
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:30
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:36
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:35
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:35
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:36
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392
@@ -5212,16 +5210,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:22
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:27
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:28
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:26
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:27
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:25
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:25
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:26
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:21
@@ -5232,18 +5230,18 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v57 offset:14
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:19
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:20
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:18
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:19
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:17
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:17
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:18
; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:16
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
@@ -5257,19 +5255,19 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400
; ALIGNED-NEXT: flat_store_byte v[5:6], v74 offset:7
; ALIGNED-NEXT: flat_store_byte v[5:6], v77 offset:8
-; ALIGNED-NEXT: flat_store_byte v[5:6], v91 offset:10
; ALIGNED-NEXT: flat_store_byte v[5:6], v76 offset:6
+; ALIGNED-NEXT: flat_store_byte v[5:6], v91 offset:11
; ALIGNED-NEXT: flat_store_byte v[5:6], v90 offset:12
-; ALIGNED-NEXT: flat_store_byte v[5:6], v92 offset:11
-; ALIGNED-NEXT: flat_store_byte v[5:6], v94 offset:9
+; ALIGNED-NEXT: flat_store_byte v[5:6], v93 offset:9
+; ALIGNED-NEXT: flat_store_byte v[5:6], v94 offset:10
; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:8
; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:2
-; ALIGNED-NEXT: flat_store_byte v[5:6], v120
-; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:1
-; ALIGNED-NEXT: flat_store_byte v[5:6], v110 offset:4
-; ALIGNED-NEXT: flat_store_byte v[5:6], v123 offset:2
-; ALIGNED-NEXT: flat_store_byte v[5:6], v122 offset:3
-; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:4
+; ALIGNED-NEXT: flat_store_byte v[5:6], v122
+; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:1
+; ALIGNED-NEXT: flat_store_byte v[5:6], v111 offset:3
+; ALIGNED-NEXT: flat_store_byte v[5:6], v120 offset:4
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:4
+; ALIGNED-NEXT: flat_store_byte v[5:6], v124 offset:2
; ALIGNED-NEXT: flat_store_byte v[3:4], v126
; ALIGNED-NEXT: s_cbranch_vccnz .LBB4_1
; ALIGNED-NEXT: ; %bb.2: ; %static-memcpy-post-expansion
@@ -5405,14 +5403,14 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3] offset:64
; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[2:3] offset:32
; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[2:3] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[96:97] offset:176
-; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[96:97] offset:192
-; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[96:97] offset:144
-; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[96:97] offset:160
-; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[96:97] offset:112
-; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[96:97] offset:128
-; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[96:97] offset:96
-; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[2:3]
+; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[2:3]
+; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[96:97] offset:192
+; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[96:97] offset:160
+; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[96:97] offset:128
+; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[96:97] offset:96
; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[96:97] offset:48
; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[96:97] offset:64
; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[96:97] offset:32
@@ -5432,21 +5430,21 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; CHECK-NEXT: s_waitcnt vmcnt(12) lgkmcnt(15)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[20:23] offset:176
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
; CHECK-NEXT: s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[24:27] offset:192
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[24:27] offset:176
; CHECK-NEXT: s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[28:31] offset:144
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[28:31] offset:192
; CHECK-NEXT: s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[32:35] offset:160
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[32:35] offset:144
; CHECK-NEXT: s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[36:39] offset:112
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[36:39] offset:160
; CHECK-NEXT: s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[48:51] offset:128
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[48:51] offset:112
; CHECK-NEXT: s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[52:55] offset:96
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[52:55] offset:128
; CHECK-NEXT: s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[64:67]
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[64:67] offset:96
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, 0x100, v0
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
@@ -5637,22 +5635,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:300
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:296
-; ALIGNED-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:300
-; ALIGNED-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288
-; ALIGNED-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v27 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v27 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v26 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v26 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v25 offset:28
-; ALIGNED-NEXT: flat_store_byte v[0:1], v25 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v25 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v24 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v24 offset:28
+; ALIGNED-NEXT: flat_store_byte v[0:1], v24 offset:32
; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:248
@@ -5964,12 +5962,12 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v25
; ALIGNED-NEXT: v_lshrrev_b32_e32 v25, 8, v25
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v31
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v49, 8, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v30
-; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v24, 24, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v48, 8, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v30, 8, v30
@@ -5977,22 +5975,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v29, 8, v29
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v28
; ALIGNED-NEXT: v_lshrrev_b32_e32 v28, 8, v28
-; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 24, v39
; ALIGNED-NEXT: v_lshrrev_b32_e32 v39, 8, v39
-; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v27, 24, v38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v38, 8, v38
-; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 24, v37
; ALIGNED-NEXT: v_lshrrev_b32_e32 v37, 8, v37
-; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v26, 24, v36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v36, 8, v36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v51
; ALIGNED-NEXT: v_lshrrev_b32_e32 v51, 8, v51
-; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v25, 24, v50
; ALIGNED-NEXT: v_lshrrev_b32_e32 v50, 8, v50
; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:28
@@ -6276,22 +6274,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:552
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:556
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:552
-; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:556
-; ALIGNED-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:544
-; ALIGNED-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:556
+; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:552
+; ALIGNED-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:544
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v37 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v37 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v31 offset:28
-; ALIGNED-NEXT: flat_store_byte v[16:17], v31 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v31 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v30 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v30 offset:28
+; ALIGNED-NEXT: flat_store_byte v[16:17], v30 offset:32
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:504
@@ -6603,12 +6601,12 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v65
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v65, 8, v65
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v64
-; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v30, 24, v52
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 8, v52
; ALIGNED-NEXT: v_lshrrev_b32_e32 v64, 8, v64
@@ -6616,22 +6614,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v51, 8, v51
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v50
; ALIGNED-NEXT: v_lshrrev_b32_e32 v50, 8, v50
-; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 24, v87
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v87
-; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v37, 24, v85
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v85
-; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 24, v71
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v71
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 24, v66
; ALIGNED-NEXT: v_lshrrev_b32_e32 v66, 8, v66
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v86
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 8, v86
-; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 24, v80
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 8, v80
; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:28
@@ -6859,19 +6857,19 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; UNROLL3-NEXT: .LBB5_2: ; %memmove_fwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: flat_load_dwordx4 v[8:11], v[4:5]
-; UNROLL3-NEXT: flat_load_dwordx4 v[12:15], v[4:5] offset:16
-; UNROLL3-NEXT: flat_load_dwordx4 v[16:19], v[4:5] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[8:11], v[4:5] offset:16
+; UNROLL3-NEXT: flat_load_dwordx4 v[12:15], v[4:5] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[16:19], v[4:5]
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, v4, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 0xffffffd0
; UNROLL3-NEXT: s_addc_u32 s5, s5, -1
; UNROLL3-NEXT: s_waitcnt vmcnt(2) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[8:11]
+; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[8:11] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[12:15] offset:16
+; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[12:15] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[16:19] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[16:19]
; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v6, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
; UNROLL3-NEXT: s_cmp_lg_u64 s[4:5], 0
@@ -6905,19 +6903,19 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; UNROLL3-NEXT: .LBB5_6: ; %memmove_bwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: flat_load_dwordx4 v[6:9], v[2:3]
-; UNROLL3-NEXT: flat_load_dwordx4 v[10:13], v[2:3] offset:16
-; UNROLL3-NEXT: flat_load_dwordx4 v[14:17], v[2:3] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[6:9], v[2:3] offset:16
+; UNROLL3-NEXT: flat_load_dwordx4 v[10:13], v[2:3] offset:32
+; UNROLL3-NEXT: flat_load_dwordx4 v[14:17], v[2:3]
; UNROLL3-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffffd0, v2
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: s_waitcnt vmcnt(2) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[6:9]
+; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[6:9] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[10:13] offset:16
+; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[10:13] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(2)
-; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[14:17] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[14:17]
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, 0xffffffd0, v4
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, vcc_lo
; UNROLL3-NEXT: s_cmp_eq_u64 s[4:5], 0
@@ -7411,22 +7409,22 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296
-; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:300
-; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288
-; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v15, off offset:42
-; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v15, off offset:46
+; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v14, off offset:46
-; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v14, off offset:42
+; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v13, off offset:34
-; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v13, off offset:38
+; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v12, off offset:38
-; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v12, off offset:34
+; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:32
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248
@@ -7665,14 +7663,14 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_store_byte v[0:1], v97, off offset:53
; ALIGNED-NEXT: global_store_byte v[0:1], v38, off offset:51
; ALIGNED-NEXT: global_store_byte v[0:1], v96, off offset:49
-; ALIGNED-NEXT: global_store_byte v[0:1], v115, off offset:43
-; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:41
-; ALIGNED-NEXT: global_store_byte v[0:1], v37, off offset:47
-; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:45
-; ALIGNED-NEXT: global_store_byte v[0:1], v20, off offset:35
-; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:33
-; ALIGNED-NEXT: global_store_byte v[0:1], v36, off offset:39
-; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:37
+; ALIGNED-NEXT: global_store_byte v[0:1], v115, off offset:47
+; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:45
+; ALIGNED-NEXT: global_store_byte v[0:1], v37, off offset:43
+; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:41
+; ALIGNED-NEXT: global_store_byte v[0:1], v20, off offset:39
+; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:37
+; ALIGNED-NEXT: global_store_byte v[0:1], v36, off offset:35
+; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:33
; ALIGNED-NEXT: global_store_byte v[0:1], v100, off offset:31
; ALIGNED-NEXT: global_store_byte v[0:1], v11, off offset:29
; ALIGNED-NEXT: global_store_byte v[0:1], v51, off offset:27
@@ -8041,22 +8039,22 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:552
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:556
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:552
-; ALIGNED-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:556
-; ALIGNED-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:544
-; ALIGNED-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:556
+; ALIGNED-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:552
+; ALIGNED-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:544
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v11, off offset:42
-; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v11, off offset:46
+; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v10, off offset:46
-; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v10, off offset:42
+; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v9, off offset:34
-; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v9, off offset:38
+; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v8, off offset:38
-; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v8, off offset:34
+; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:32
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:504
@@ -8295,14 +8293,14 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_store_byte v[12:13], v18, off offset:53
; ALIGNED-NEXT: global_store_byte v[12:13], v84, off offset:51
; ALIGNED-NEXT: global_store_byte v[12:13], v16, off offset:49
-; ALIGNED-NEXT: global_store_byte v[12:13], v115, off offset:43
-; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:41
-; ALIGNED-NEXT: global_store_byte v[12:13], v69, off offset:47
-; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:45
-; ALIGNED-NEXT: global_store_byte v[12:13], v50, off offset:35
-; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:33
-; ALIGNED-NEXT: global_store_byte v[12:13], v54, off offset:39
-; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:37
+; ALIGNED-NEXT: global_store_byte v[12:13], v115, off offset:47
+; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:45
+; ALIGNED-NEXT: global_store_byte v[12:13], v69, off offset:43
+; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:41
+; ALIGNED-NEXT: global_store_byte v[12:13], v50, off offset:39
+; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:37
+; ALIGNED-NEXT: global_store_byte v[12:13], v54, off offset:35
+; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:33
; ALIGNED-NEXT: global_store_byte v[12:13], v100, off offset:31
; ALIGNED-NEXT: global_store_byte v[12:13], v7, off offset:29
; ALIGNED-NEXT: global_store_byte v[12:13], v85, off offset:27
@@ -8375,19 +8373,19 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; UNROLL3-NEXT: .LBB6_2: ; %memmove_fwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[4:5], off
-; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[4:5], off offset:16
-; UNROLL3-NEXT: global_load_dwordx4 v[16:19], v[4:5], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[4:5], off offset:16
+; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[4:5], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[16:19], v[4:5], off
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, v4, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 0xffffffd0
; UNROLL3-NEXT: s_addc_u32 s5, s5, -1
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
-; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[8:11], off
+; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[12:15], off offset:16
+; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[12:15], off offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[16:19], off offset:32
+; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[16:19], off
; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v6, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
; UNROLL3-NEXT: s_cmp_lg_u64 s[4:5], 0
@@ -8421,19 +8419,19 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; UNROLL3-NEXT: .LBB6_6: ; %memmove_bwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
-; UNROLL3-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; UNROLL3-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16
-; UNROLL3-NEXT: global_load_dwordx4 v[14:17], v[2:3], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[6:9], v[2:3], off offset:16
+; UNROLL3-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[14:17], v[2:3], off
; UNROLL3-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffffd0, v2
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
-; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[6:9], off
+; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:16
+; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[14:17], off offset:32
+; UNROLL3-NEXT: global_store_dwordx4 v[4:5], v[14:17], off
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, 0xffffffd0, v4
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, vcc_lo
; UNROLL3-NEXT: s_cmp_eq_u64 s[4:5], 0
@@ -8750,13 +8748,13 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v55
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v55
; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:200
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v53
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:198
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v53
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:204
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v52
; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:202
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v52
; ALIGNED-NEXT: flat_store_byte v[98:99], v80 offset:196
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v50
@@ -8799,18 +8797,18 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 24, v36
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:174
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 8, v36
-; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v54 offset:164
-; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v55 offset:168
; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:169
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:157
+; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v53 offset:160
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:161
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v53, 8, v34
@@ -8824,10 +8822,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v32
; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:170
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:160
-; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:158
-; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:164
-; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:162
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:164
+; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:162
+; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:160
+; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:158
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:200
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:204
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:196
@@ -9135,9 +9133,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v101, off, s[0:3], s32 offset:292
; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v102
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v102
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v103
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v103
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v102 offset:244
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v103 offset:248
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:249
@@ -9146,6 +9141,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v101 offset:241
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v100 offset:236
; ALIGNED-NEXT: flat_store_byte v[98:99], v100 offset:237
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v102
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v103
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v103
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v101
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 8, v101
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v100
@@ -9171,6 +9169,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:316
; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:308
; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:232
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v81
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v86 offset:228
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v87 offset:232
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:233
@@ -9179,13 +9180,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:225
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v84 offset:220
; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:221
-; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v82
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v82
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v83
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v83
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:232
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v81
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:230
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v81
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:236
@@ -9217,8 +9215,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v69
; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v68
; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v68
-; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:216
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:212
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v65
+; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:216
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v66
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:214
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v66
@@ -9226,8 +9226,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v67
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:218
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v67
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:212
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v65
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:210
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v65
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:208
@@ -9284,8 +9282,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v65, 8, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v66, 24, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v48
-; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:184
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:180
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v37
+; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:184
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v38
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:182
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v38
@@ -9293,8 +9293,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v39
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:186
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v39
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:180
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v37
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:178
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v37
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:176
@@ -9365,6 +9363,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:348
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:340
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:132
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v18
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v26
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v38 offset:132
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v39 offset:136
; ALIGNED-NEXT: flat_store_byte v[98:99], v39 offset:137
@@ -9377,21 +9379,14 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:134
; ALIGNED-NEXT: flat_store_byte v[98:99], v86 offset:140
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:138
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:132
-; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v18
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:130
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v18
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:128
; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:126
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:424
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:428
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:420
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:120
-; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v14
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v26
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v18
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v34 offset:116
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v35 offset:120
; ALIGNED-NEXT: flat_store_byte v[98:99], v35 offset:121
@@ -9400,6 +9395,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v33 offset:113
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v32 offset:108
; ALIGNED-NEXT: flat_store_byte v[98:99], v32 offset:109
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:120
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v14
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:118
; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:124
; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:122
@@ -9452,10 +9450,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v34, 24, v16
; ALIGNED-NEXT: v_lshrrev_b32_e32 v35, 8, v16
; ALIGNED-NEXT: v_lshrrev_b32_e32 v55, 8, v15
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 24, v12
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v12
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v13
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 24, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v12
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v12
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v11
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:104
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v9
@@ -9528,26 +9526,26 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v16 offset:44
; ALIGNED-NEXT: flat_store_byte v[98:99], v35 offset:46
; ALIGNED-NEXT: flat_store_byte v[98:99], v16 offset:45
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:508
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:508
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:496
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:40
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v14 offset:36
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:40
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:38
-; ALIGNED-NEXT: flat_store_byte v[98:99], v14 offset:37
-; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:44
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v15 offset:40
-; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:42
+; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:44
; ALIGNED-NEXT: flat_store_byte v[98:99], v15 offset:41
-; ALIGNED-NEXT: flat_store_byte v[98:99], v68 offset:32
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v12 offset:28
-; ALIGNED-NEXT: flat_store_byte v[98:99], v69 offset:30
-; ALIGNED-NEXT: flat_store_byte v[0:1], v12 offset:32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v70 offset:36
+; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:42
+; ALIGNED-NEXT: flat_store_byte v[98:99], v14 offset:37
+; ALIGNED-NEXT: flat_store_byte v[98:99], v68 offset:36
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v13 offset:32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:34
+; ALIGNED-NEXT: flat_store_byte v[98:99], v69 offset:34
; ALIGNED-NEXT: flat_store_byte v[98:99], v13 offset:33
+; ALIGNED-NEXT: flat_store_byte v[98:99], v70 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v12 offset:28
+; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:30
+; ALIGNED-NEXT: flat_store_byte v[0:1], v12 offset:32
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:456
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:460
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:452
@@ -9618,8 +9616,8 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
; UNROLL3-NEXT: global_load_dwordx4 v[8:11], v[4:5], off offset:16
-; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[4:5], off
-; UNROLL3-NEXT: global_load_dwordx4 v[16:19], v[4:5], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[12:15], v[4:5], off offset:32
+; UNROLL3-NEXT: global_load_dwordx4 v[16:19], v[4:5], off
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, v4, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 0xffffffd0
@@ -9627,9 +9625,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[8:11] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[12:15]
+; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[12:15] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[16:19] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[16:19]
; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v6, 48
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
; UNROLL3-NEXT: s_cmp_lg_u64 s[4:5], 0
@@ -9664,8 +9662,8 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0x2
; UNROLL3-NEXT: global_load_dwordx4 v[6:9], v[2:3], off offset:1984
-; UNROLL3-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:1968
-; UNROLL3-NEXT: global_load_dwordx4 v[14:17], v[2:3], off offset:2000
+; UNROLL3-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:2000
+; UNROLL3-NEXT: global_load_dwordx4 v[14:17], v[2:3], off offset:1968
; UNROLL3-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffffd0, v2
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
@@ -9673,9 +9671,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; UNROLL3-NEXT: s_waitcnt vmcnt(2)
; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[6:9] offset:16
; UNROLL3-NEXT: s_waitcnt vmcnt(1)
-; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[10:13]
+; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[10:13] offset:32
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
-; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[14:17] offset:32
+; UNROLL3-NEXT: flat_store_dwordx4 v[4:5], v[14:17]
; UNROLL3-NEXT: v_add_co_u32 v4, vcc_lo, 0xffffffd0, v4
; UNROLL3-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, vcc_lo
; UNROLL3-NEXT: s_cmp_eq_u64 s[4:5], 0
@@ -12768,11 +12766,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_mov_b32_e32 v88, v1
-; ALIGNED-NEXT: v_mov_b32_e32 v87, v0
+; ALIGNED-NEXT: v_mov_b32_e32 v81, v1
+; ALIGNED-NEXT: v_mov_b32_e32 v80, v0
; ALIGNED-NEXT: s_mov_b32 s4, exec_lo
-; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[87:88]
-; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v87, vcc_lo
+; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[80:81]
+; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v80, vcc_lo
; ALIGNED-NEXT: v_cmpx_ge_u32_e32 v2, v3
; ALIGNED-NEXT: s_xor_b32 s6, exec_lo, s4
; ALIGNED-NEXT: s_cbranch_execz .LBB9_3
@@ -12783,155 +12781,154 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_clause 0x3a
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:20
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:22
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:23
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:22
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:24
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:25
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:26
; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:27
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:28
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:29
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:31
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:27
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:33
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:35
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:36
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:37
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:38
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:39
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:40
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:41
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:42
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:43
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:44
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:45
-; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:46
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:42
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:47
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:48
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:49
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:43
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:51
-; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:52
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:53
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:52
+; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:54
; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:55
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:56
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:57
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:58
-; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:60
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:61
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:62
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:63
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:64
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:65
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:66
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:66
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:59
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:67
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:67
; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:68
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:69
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:70
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:71
-; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:72
-; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:73
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:74
-; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:76
-; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:77
-; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:78
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:69
+; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:70
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:71
+; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:72
+; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:73
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:76
+; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:78
; ALIGNED-NEXT: s_add_u32 s4, s4, 0xffffff00
; ALIGNED-NEXT: s_addc_u32 s5, s5, -1
; ALIGNED-NEXT: s_cmp_lg_u64 s[4:5], 0
; ALIGNED-NEXT: s_waitcnt vmcnt(58)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(57)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
; ALIGNED-NEXT: s_waitcnt vmcnt(56)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(55)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v4
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(55)
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(54)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(53)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v9, 8, v7
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(52)
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(50)
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: s_waitcnt vmcnt(49)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(48)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v8, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v6, 8, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(46)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v13, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v10, 8, v13
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v9, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v12, 8, v11
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(44)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v15
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v14
; ALIGNED-NEXT: s_waitcnt vmcnt(42)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v20, 8, v18
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
; ALIGNED-NEXT: s_waitcnt vmcnt(40)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v17, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v15, 8, v19
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(38)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v21, 8, v19
-; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v20, 8, v21
+; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(36)
-; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
+; ALIGNED-NEXT: s_waitcnt vmcnt(33)
+; ALIGNED-NEXT: v_lshl_or_b32 v13, v22, 8, v23
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v10, 16, v9
-; ALIGNED-NEXT: s_waitcnt vmcnt(34)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v28, 8, v26
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(31)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(32)
-; ALIGNED-NEXT: v_lshl_or_b32 v15, v25, 8, v22
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v15, v27, 8, v24
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v12, 16, v11
-; ALIGNED-NEXT: s_waitcnt vmcnt(30)
-; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v27
-; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: s_waitcnt vmcnt(27)
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v28, 8, v29
; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v30
; ALIGNED-NEXT: s_waitcnt vmcnt(26)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v33
; ALIGNED-NEXT: s_waitcnt vmcnt(24)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v37, 8, v32
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v32, 8, v35
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v14, 16, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(22)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v36, 8, v35
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v36, 8, v37
; ALIGNED-NEXT: s_waitcnt vmcnt(17)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v38
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v38, 8, v39
; ALIGNED-NEXT: s_waitcnt vmcnt(15)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v49, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v50, 8, v49
; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v48
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v16, 16, v15
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v52, 8, v53
; ALIGNED-NEXT: v_lshl_or_b32 v4, v55, 8, v54
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v64, 8, v66
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v66, 8, v67
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_clause 0x4
@@ -12943,63 +12940,63 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v3, 16, v9
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v65
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v64, 8, v65
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v5, 16, v4
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v69, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v68, 8, v69
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:81
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:75
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v70
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v70, 8, v71
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v65, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v65, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v70, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:91
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:113
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:113
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:114
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:115
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:116
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:116
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
@@ -13011,151 +13008,152 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v0, 8, v80
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v0, 8, v82
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v82, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v83, 8, v84
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v84, 8, v83
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v85, 8, v86
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v1
-; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:85
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:88
; ALIGNED-NEXT: v_lshl_or_b32 v4, v7, 8, v6
+; ALIGNED-NEXT: s_clause 0x1
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:89
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:87
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:89
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:86
+; ALIGNED-NEXT: s_clause 0x1
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:87
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:88
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:84
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:94
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:93
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:94
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:95
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:92
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:103
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:100
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:100
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:97
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 8, v7
; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:98
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:105
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:107
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:98
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:106
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:97
+; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:99
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:106
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:107
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:96
-; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:104
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v6
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:105
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v3, 8, v0
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:102
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:102
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:103
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:101
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:111
-; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v1
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:109
+; ALIGNED-NEXT: s_waitcnt vmcnt(1)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:110
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:110
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:111
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:108
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:120
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:122
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:118
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 8, v7
; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
; ALIGNED-NEXT: s_clause 0x3
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:120
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:121
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:122
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:123
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:124
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:123
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:124
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: s_clause 0x3
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:128
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:129
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:129
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:130
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:131
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
@@ -13164,27 +13162,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:117
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:126
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:132
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:132
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:127
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
@@ -13192,23 +13190,23 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:125
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:134
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:136
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
@@ -13216,433 +13214,438 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:133
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x6
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:137
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:138
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:139
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:140
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:138
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:139
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:140
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:142
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:144
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:145
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:145
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_clause 0x3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:143
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:146
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:147
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:148
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:148
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:141
-; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:160
-; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:161
-; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:162
-; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:163
-; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:164
-; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:167
-; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:168
-; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:169
-; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:170
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:172
-; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:173
-; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:174
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:175
-; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:176
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:177
-; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:178
-; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:180
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:181
-; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:182
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:183
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:184
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:185
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:186
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:188
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:190
-; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:191
-; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:192
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:193
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:194
-; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:195
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:196
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:197
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:198
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:199
-; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:200
-; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:201
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:202
-; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:204
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:205
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:206
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:207
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:208
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:209
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:210
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:212
-; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:229
-; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:230
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:231
-; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:232
-; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:233
-; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:234
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:235
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:236
-; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:237
; ALIGNED-NEXT: s_clause 0x17
-; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:238
-; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:239
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:240
-; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:241
-; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:242
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:243
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:244
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:245
-; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:246
-; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:247
-; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:248
-; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:249
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:250
-; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:253
-; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:254
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:255
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen
+; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:2
; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:3
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:4
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:5
-; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:6
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:7
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:150
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:156
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:156
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:152
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:154
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:154
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:153
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:155
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:151
; ALIGNED-NEXT: s_waitcnt vmcnt(21)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v46, 8, v56
-; ALIGNED-NEXT: s_waitcnt vmcnt(13)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v101, 8, v112
-; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v79, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v43, 8, v42
+; ALIGNED-NEXT: s_waitcnt vmcnt(15)
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v112, 8, v102
+; ALIGNED-NEXT: s_waitcnt vmcnt(12)
+; ALIGNED-NEXT: buffer_store_dword v89, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:149
-; ALIGNED-NEXT: buffer_store_dword v92, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v77, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(11)
+; ALIGNED-NEXT: buffer_store_dword v91, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(9)
+; ALIGNED-NEXT: buffer_store_dword v72, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: buffer_store_dword v75, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:158
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:213
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:214
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:215
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:216
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:217
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:218
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:219
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:159
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:157
-; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
+; ALIGNED-NEXT: s_clause 0x7
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:220
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v123
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v97, 8, v95
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v106, 8, v109
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v79, 8, v77
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v57, 8, v107
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v47, 8, v59
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v59, 8, v93
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v47, 8, v56
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v100, 8, v113
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 8, v102
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v113, 8, v95
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v100, 8, v103
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v43, 8, v45
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v117, 8, v115
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v41, 8, v44
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v119, 8, v117
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v85
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v81
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v85, 8, v87
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v71
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v69, 8, v84
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v68, 8, v70
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v68, 8, v82
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v67
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v55
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v54
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v51, 8, v53
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v50, 8, v49
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v66, 8, v67
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v64, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v64, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v54, 8, v55
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v48, 8, v51
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v38, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v39, 8, v48
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v37
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:221
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:222
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:223
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:224
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:225
-; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:226
-; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:227
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:228
-; ALIGNED-NEXT: v_lshl_or_b32 v74, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v35, 8, v49
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v36
-; ALIGNED-NEXT: v_lshl_or_b32 v41, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v25, 8, v28
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v24, 8, v26
-; ALIGNED-NEXT: v_lshl_or_b32 v80, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v33
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v29
-; ALIGNED-NEXT: v_lshl_or_b32 v71, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v18
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v11, 8, v14
-; ALIGNED-NEXT: v_lshl_or_b32 v50, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v15, 8, v17
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v12
-; ALIGNED-NEXT: v_lshl_or_b32 v37, v4, 16, v3
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:227
+; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT: v_lshl_or_b32 v63, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v34, 8, v38
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v32, 8, v33
+; ALIGNED-NEXT: v_lshl_or_b32 v115, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v24, 8, v26
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v23
+; ALIGNED-NEXT: v_lshl_or_b32 v83, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v29, 8, v31
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v28, 8, v27
+; ALIGNED-NEXT: v_lshl_or_b32 v69, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v18, 8, v17
+; ALIGNED-NEXT: v_lshl_or_b32 v52, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(15)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v15, 8, v20
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
+; ALIGNED-NEXT: v_lshl_or_b32 v35, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v16
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v8
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v8
-; ALIGNED-NEXT: v_lshl_or_b32 v32, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v21, 8, v23
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v20, 8, v22
-; ALIGNED-NEXT: v_lshl_or_b32 v27, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v30, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v11, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v25, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v126, 8, v0
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v126
-; ALIGNED-NEXT: v_lshl_or_b32 v19, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v122, 8, v124
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v1
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v110, 8, v120
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v78, 8, v94
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v90, 8, v89
-; ALIGNED-NEXT: v_lshl_or_b32 v62, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v106, 8, v108
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v104
-; ALIGNED-NEXT: v_lshl_or_b32 v60, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v75, 8, v91
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v72, 8, v76
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v110, 8, v120
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v108, 8, v107
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v88, 8, v90
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v76, 8, v78
+; ALIGNED-NEXT: v_lshl_or_b32 v58, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v104
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v92, 8, v94
+; ALIGNED-NEXT: v_lshl_or_b32 v45, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v74, 8, v73
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v60, 8, v62
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v58, 8, v73
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v103, 16, v4
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v114, 8, v116
-; ALIGNED-NEXT: v_lshl_or_b32 v98, v119, 16, v103
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v42, 8, v44
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v118, 8, v40
-; ALIGNED-NEXT: v_lshl_or_b32 v99, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v46, 8, v57
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v114, 16, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v97, 8, v99
+; ALIGNED-NEXT: v_lshl_or_b32 v96, v114, 16, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v40, 8, v118
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v101, 8, v116
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v114, 16, v98
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:1
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen
+; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:1
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v119, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v103, 8, v96
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v61, 8, v119
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v79, 8, v77
+; ALIGNED-NEXT: buffer_store_dword v114, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v114, 8, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v61, 8, v89
; ALIGNED-NEXT: s_clause 0x4
-; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:12
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:13
; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:14
; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:15
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:11
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v63, 8, v92
+; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v114, 16, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v75, 8, v72
+; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v84, 8, v91
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:8
-; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:9
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:10
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v114, 16, v98
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v109, 8, v125
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v125, 8, v123
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v111, 8, v121
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v111, 8, v121
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v114, 16, v98
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v77, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v75, 8, v91
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v79, 8, v63
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v89, 8, v72
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v114, 16, v98
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:18
; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:17
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:236
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224
-; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v87, 3
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v88, vcc_lo
-; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:247
-; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:248
-; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:246
-; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:252
-; ALIGNED-NEXT: flat_store_byte v[3:4], v114 offset:250
-; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:251
-; ALIGNED-NEXT: flat_store_byte v[3:4], v116 offset:249
-; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:245
-; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:239
-; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:238
-; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:244
-; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:242
-; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:243
-; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:241
-; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:237
-; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:231
-; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:232
-; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:230
-; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:236
-; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:234
-; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:235
-; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:233
-; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:229
-; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:223
-; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:224
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:222
-; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:228
-; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:226
-; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:227
-; ALIGNED-NEXT: flat_store_byte v[3:4], v1 offset:225
-; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:221
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:192
-; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:206
-; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:208
-; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:207
-; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:210
-; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:212
-; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:211
-; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:209
-; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:215
+; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v80, 3
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v81, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[3:4], v116 offset:247
+; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:248
+; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:246
+; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:251
+; ALIGNED-NEXT: flat_store_byte v[3:4], v97 offset:252
+; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:249
+; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:250
+; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:245
+; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:239
+; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:238
+; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:243
+; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:244
+; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:241
+; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:242
+; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:237
+; ALIGNED-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:231
+; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:232
+; ALIGNED-NEXT: flat_store_byte v[3:4], v105 offset:230
+; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:235
+; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:236
+; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:233
+; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:234
+; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:229
+; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:223
+; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:224
+; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:222
+; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:227
+; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:228
+; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:225
+; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:226
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:221
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:215
; ALIGNED-NEXT: flat_store_byte v[3:4], v10 offset:216
-; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:219
; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:220
+; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:217
; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:218
-; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:219
-; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:217
-; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:213
-; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:205
-; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v74, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:207
+; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:206
+; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:211
+; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:209
+; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:210
+; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:205
+; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v115, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:199
-; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:200
-; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:198
-; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:204
-; ALIGNED-NEXT: flat_store_byte v[3:4], v25 offset:202
-; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:203
-; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:201
-; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:197
-; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:191
-; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:192
-; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:190
-; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v35 offset:194
-; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:195
-; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:193
-; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:189
+; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:199
+; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:200
+; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:198
+; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:203
+; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:204
+; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:201
+; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:202
+; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:197
+; ALIGNED-NEXT: flat_store_byte v[3:4], v37 offset:191
+; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:192
+; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:190
+; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:195
+; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:196
+; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:193
+; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:194
+; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:189
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v127, 8, v119
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v127, 8, v98
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v127, v103, 8, v61
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v96, 16, v127
+; ALIGNED-NEXT: v_lshl_or_b32 v127, v114, 8, v61
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v84, 16, v127
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
@@ -13655,22 +13658,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:183
-; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:184
-; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:182
-; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:188
-; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:186
-; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:187
-; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:185
-; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:181
-; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:175
-; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:176
-; ALIGNED-NEXT: flat_store_byte v[3:4], v83 offset:174
-; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:180
-; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:178
-; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:179
-; ALIGNED-NEXT: flat_store_byte v[3:4], v84 offset:177
-; ALIGNED-NEXT: flat_store_byte v[3:4], v85 offset:173
+; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:183
+; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:184
+; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:182
+; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:187
+; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:188
+; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:185
+; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:186
+; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:181
+; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:175
+; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:176
+; ALIGNED-NEXT: flat_store_byte v[3:4], v85 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:179
+; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:180
+; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:177
+; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:178
+; ALIGNED-NEXT: flat_store_byte v[3:4], v87 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
@@ -13682,23 +13685,23 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:167
-; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:168
-; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:166
-; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:172
-; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:170
-; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:171
-; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:169
-; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:165
-; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:159
-; ALIGNED-NEXT: flat_store_byte v[3:4], v97 offset:160
-; ALIGNED-NEXT: flat_store_byte v[3:4], v105 offset:158
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:167
+; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:168
+; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:166
+; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:171
+; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:172
+; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:169
+; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:170
+; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:165
+; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:159
+; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:160
+; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:158
+; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:163
; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:164
-; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:162
-; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:163
-; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:161
-; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:157
+; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:161
+; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:162
+; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
@@ -13719,18 +13722,18 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:150
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:155
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:156
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:154
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:155
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:153
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:153
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:154
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:149
@@ -13743,18 +13746,18 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:142
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:147
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:148
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:146
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:147
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:145
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:145
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:146
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:141
@@ -13781,16 +13784,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:134
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:140
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:138
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:137
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:137
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:138
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:133
@@ -13805,19 +13808,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:126
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:132
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:130
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:130
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:128
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:128
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360
@@ -13841,16 +13844,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:118
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:124
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:122
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:121
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:121
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:122
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:117
@@ -13865,16 +13868,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:110
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:116
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:114
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:113
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:113
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:114
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:109
@@ -13901,16 +13904,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:102
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:108
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:106
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:105
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:105
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:106
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:101
@@ -13925,16 +13928,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:94
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:100
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:98
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:97
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:97
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:98
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:93
@@ -13944,7 +13947,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
@@ -13961,16 +13964,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:86
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:92
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:90
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:89
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:89
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:90
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:85
@@ -13985,16 +13988,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:78
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:84
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:82
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:81
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:81
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:82
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:77
@@ -14004,7 +14007,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
@@ -14021,23 +14024,23 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:70
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:76
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:74
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:75
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:75
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:76
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:73
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:74
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:69
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:63
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
@@ -14045,19 +14048,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:62
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:68
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:66
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:67
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:67
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:68
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:66
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:64
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424
@@ -14067,13 +14070,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
@@ -14084,46 +14084,49 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:54
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:60
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:57
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:57
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:46
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:52
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:49
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:49
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436
@@ -14132,52 +14135,52 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:39
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:31
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:36
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392
@@ -14187,7 +14190,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:384
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:23
@@ -14199,90 +14202,90 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:22
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:28
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:26
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:25
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:25
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:26
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:21
-; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:15
+; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:15
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:16
-; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:14
+; ALIGNED-NEXT: flat_store_byte v[3:4], v114 offset:14
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:20
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:18
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[87:88], v61 offset:16
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:18
+; ALIGNED-NEXT: flat_store_byte v[80:81], v61 offset:16
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:7
-; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:8
-; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:10
-; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:6
-; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:12
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:7
+; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:8
+; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:6
; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:11
-; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:9
-; ALIGNED-NEXT: flat_store_byte v[87:88], v92 offset:8
+; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:12
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:9
+; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:10
+; ALIGNED-NEXT: flat_store_byte v[80:81], v91 offset:8
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:2
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:2
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:1
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:1
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:4
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0 offset:4
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_co_u32 v87, vcc_lo, 0x100, v87
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v88, null, 0, v88, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[80:81], v0
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
+; ALIGNED-NEXT: v_add_co_u32 v80, vcc_lo, 0x100, v80
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v81, null, 0, v81, vcc_lo
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:3
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:4
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:2
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:3
; ALIGNED-NEXT: s_cbranch_scc1 .LBB9_2
; ALIGNED-NEXT: .LBB9_3: ; %Flow16
; ALIGNED-NEXT: s_andn2_saveexec_b32 s6, s6
; ALIGNED-NEXT: s_cbranch_execz .LBB9_6
; ALIGNED-NEXT: ; %bb.4: ; %memmove_bwd_loop.preheader
-; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0x700, v87
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, 0, v88, vcc_lo
+; ALIGNED-NEXT: v_add_co_u32 v114, vcc_lo, 0x700, v80
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v115, null, 0, v81, vcc_lo
; ALIGNED-NEXT: s_movk_i32 s4, 0xf800
; ALIGNED-NEXT: s_mov_b32 s5, -1
; ALIGNED-NEXT: .LBB9_5: ; %memmove_bwd_loop
@@ -14290,48 +14293,48 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_clause 0x3b
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1812
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1813
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1814
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1815
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1816
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1814
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1815
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1816
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1817
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1818
-; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:1811
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1819
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1820
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1821
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1822
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1823
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:1824
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1818
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:1811
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1819
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1820
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1821
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1822
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1823
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1824
; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:1825
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:1826
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:1827
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1828
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:1829
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1826
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:1827
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:1828
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:1829
; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:1830
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:1831
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:1832
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:1833
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:1834
-; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1835
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:1836
-; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:1837
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:1838
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:1839
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:1831
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:1832
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:1833
+; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1834
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:1836
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:1837
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:1838
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:1839
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:1840
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1841
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1842
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1843
+; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1842
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:1835
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1843
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:1844
-; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:1845
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1846
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1847
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:1848
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1845
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1846
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:1847
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1848
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:1849
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:1850
; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:1852
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1853
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1854
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1855
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:1853
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:1854
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1855
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:1856
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:1857
; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:1858
@@ -14352,234 +14355,233 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
; ALIGNED-NEXT: s_cmp_eq_u64 s[4:5], 0
; ALIGNED-NEXT: s_waitcnt vmcnt(59)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(58)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
; ALIGNED-NEXT: s_waitcnt vmcnt(57)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(56)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v4
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(56)
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(55)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(54)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v9, 8, v7
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(53)
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(51)
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: s_waitcnt vmcnt(50)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(49)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v8, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v6, 8, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(47)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v13, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v10, 8, v13
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v9, 8, v8
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v12, 8, v11
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(45)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v15
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v14
; ALIGNED-NEXT: s_waitcnt vmcnt(43)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v20, 8, v18
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
; ALIGNED-NEXT: s_waitcnt vmcnt(41)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v19, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v15, 8, v19
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(39)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v22, 8, v21
-; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v20, 8, v21
+; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(37)
-; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
+; ALIGNED-NEXT: s_waitcnt vmcnt(34)
+; ALIGNED-NEXT: v_lshl_or_b32 v13, v22, 8, v23
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v10, 16, v9
-; ALIGNED-NEXT: s_waitcnt vmcnt(35)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v30, 8, v26
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(32)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(33)
-; ALIGNED-NEXT: v_lshl_or_b32 v15, v27, 8, v25
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v15, v27, 8, v24
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v12, 16, v11
-; ALIGNED-NEXT: s_waitcnt vmcnt(31)
-; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v28
-; ALIGNED-NEXT: s_waitcnt vmcnt(29)
+; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v30
; ALIGNED-NEXT: v_lshl_or_b32 v3, v32, 8, v31
; ALIGNED-NEXT: s_waitcnt vmcnt(27)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v35, 8, v34
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v35
; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v38, 8, v33
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v34, 8, v33
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v14, 16, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(23)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v37, 8, v36
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v38, 8, v37
; ALIGNED-NEXT: s_waitcnt vmcnt(18)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v48, 8, v39
; ALIGNED-NEXT: s_waitcnt vmcnt(16)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v49, 8, v17
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v48
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v50, 8, v28
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v49
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v16, 16, v15
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(12)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v52
; ALIGNED-NEXT: v_lshl_or_b32 v4, v55, 8, v54
-; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v65, 8, v64
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
-; ALIGNED-NEXT: s_clause 0x4
+; ALIGNED-NEXT: s_clause 0x3
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1875
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1899
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1907
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1900
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1872
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1874
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v3, 16, v9
-; ALIGNED-NEXT: s_waitcnt vmcnt(14)
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v66
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1882
-; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v5, 16, v4
-; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: s_waitcnt vmcnt(12)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v69, 8, v68
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1873
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1871
-; ALIGNED-NEXT: s_waitcnt vmcnt(9)
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v82
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v65, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v70, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v112, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_clause 0x3
+; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(7)
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v113, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1883
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1904
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1905
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1906
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1908
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1907
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(7)
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(6)
+; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(5)
+; ALIGNED-NEXT: s_waitcnt vmcnt(6)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(4)
+; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v0, 8, v84
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v70
; ALIGNED-NEXT: v_lshl_or_b32 v4, v81, 8, v80
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v1
-; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1877
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1879
; ALIGNED-NEXT: v_lshl_or_b32 v4, v7, 8, v6
+; ALIGNED-NEXT: s_clause 0x1
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1878
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1880
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1880
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1879
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1881
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1878
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1876
-; ALIGNED-NEXT: s_waitcnt vmcnt(9)
+; ALIGNED-NEXT: s_waitcnt vmcnt(10)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v4
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1887
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1886
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1885
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1886
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1887
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1884
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v4
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v5
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1892
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v1, 8, v0
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1889
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
@@ -14587,7 +14589,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_clause 0x2
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1890
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1896
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1898
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1899
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_clause 0x1
@@ -14596,7 +14598,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1888
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
@@ -14604,11 +14606,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v6
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1900
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1898
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
@@ -14620,48 +14622,48 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1893
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1902
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1908
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1905
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1903
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1901
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1910
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1916
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1921
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v6
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1912
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1913
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1914
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1915
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1912
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1913
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1915
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1916
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: s_clause 0x3
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1914
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1920
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1921
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1922
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1923
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
@@ -14670,27 +14672,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1909
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(5)
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1918
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1924
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1924
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1919
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
@@ -14698,23 +14700,23 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1917
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1926
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1928
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1928
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
@@ -14722,152 +14724,141 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1925
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x6
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1929
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1930
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1931
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1932
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1929
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1930
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1931
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1932
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1934
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1936
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1937
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1937
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_clause 0x3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1935
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1938
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1939
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1940
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1940
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1933
-; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:1957
-; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:1958
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:1959
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:1960
-; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:1961
-; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:1962
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:1963
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1964
-; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:1965
-; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1966
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:1957
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:1958
+; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:1959
+; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:1960
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:1961
+; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1962
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:1963
+; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:1964
+; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1965
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:1966
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:1967
-; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:1968
-; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:1969
-; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:1970
-; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:1971
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1972
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1968
+; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:1969
+; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1970
+; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:1971
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:1972
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:1973
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:1974
+; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:1974
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:1975
-; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:1976
-; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:1977
-; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:1978
+; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:1976
+; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:1977
+; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:1978
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:1979
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:1980
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:1981
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:1982
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:1983
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:1984
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:1985
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:1986
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:1980
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:1981
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:1982
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:1983
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:1984
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:1985
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:1986
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:1987
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:1988
+; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:1988
; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1989
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1990
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:1990
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:1991
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:1992
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1993
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:1992
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1993
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1994
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1995
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1996
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1995
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:1996
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:1997
-; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1998
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:1998
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1999
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:2000
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:2000
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:2001
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:2002
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:2003
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:2004
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:2005
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:2006
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:2007
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:2008
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:2009
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:2010
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2011
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:2012
-; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:2029
-; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:2030
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:2031
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:2002
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:2003
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:2004
+; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:2029
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:2030
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:2031
; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:2032
-; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:2033
-; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:2034
-; ALIGNED-NEXT: s_clause 0x1b
-; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:2035
+; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:2033
+; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:2034
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:2035
; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:2036
-; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:2037
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:2038
-; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:2039
-; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:2040
+; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:2037
+; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:2038
+; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:2039
+; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:2040
; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:2041
-; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:2042
+; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:2042
+; ALIGNED-NEXT: s_clause 0x10
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:2043
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:2044
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:2045
; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:2046
; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:2047
-; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:1793
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:1794
-; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:1795
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:1796
-; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1797
-; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:1798
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:1799
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:1804
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:1805
-; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:1806
-; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:1807
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:1803
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:1800
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:1801
-; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:1802
+; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:1793
+; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:1794
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:1795
+; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:1796
+; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:1797
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:1798
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:1799
+; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:1804
+; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:1805
+; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:1806
+; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:1807
+; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:1803
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1942
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1948
-; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v61, 8, v72
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1948
+; ALIGNED-NEXT: s_waitcnt vmcnt(22)
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v62, 8, v61
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1944
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1946
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1944
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1946
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1945
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1947
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1945
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1947
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1953
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1953
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
@@ -14877,342 +14868,357 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1943
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v118, 8, v41
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(22)
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v118, 8, v41
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1941
-; ALIGNED-NEXT: s_waitcnt vmcnt(21)
-; ALIGNED-NEXT: buffer_store_dword v97, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(20)
+; ALIGNED-NEXT: buffer_store_dword v102, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(18)
+; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1954
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:1950
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1956
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v12
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1956
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1951
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1949
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v14, 8, v13
; ALIGNED-NEXT: s_clause 0x7
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:2005
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:2006
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:2007
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:2008
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:2009
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:2010
+; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2011
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:2012
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v95, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v90, 8, v93
+; ALIGNED-NEXT: s_clause 0x7
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:2013
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:2014
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:2014
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:2015
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:2016
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:2017
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:2018
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:2016
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:2017
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2018
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:2019
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:2020
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:2020
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v107, 8, v5
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v95
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v45, 8, v120
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v42, 8, v117
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v47, 8, v63
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v42, 8, v45
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v89, 8, v91
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v79, 8, v75
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v100, 8, v115
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v98, 8, v99
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v79, 8, v88
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v72, 8, v63
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v101
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v84
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v99, 8, v101
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v96, 8, v97
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v69
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v81
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v80, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v71
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v66, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v64, 8, v65
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v54, 8, v65
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v53
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v80
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v69
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v55
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v50
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v55
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v51
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:2021
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:2022
-; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:2023
-; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:2024
+; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:2022
+; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:2023
+; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:2024
; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:2025
-; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:2026
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:2027
-; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:2028
-; ALIGNED-NEXT: v_lshl_or_b32 v77, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v34
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v32
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v39
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v35
-; ALIGNED-NEXT: v_lshl_or_b32 v85, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v24
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v17, 8, v20
-; ALIGNED-NEXT: v_lshl_or_b32 v64, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v21, 8, v23
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v16, 8, v18
-; ALIGNED-NEXT: v_lshl_or_b32 v51, v4, 16, v3
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:2026
+; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:2027
+; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:2028
+; ALIGNED-NEXT: v_lshl_or_b32 v125, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v54
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v60, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v33
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v29
+; ALIGNED-NEXT: v_lshl_or_b32 v100, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v36, 8, v38
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v35
+; ALIGNED-NEXT: v_lshl_or_b32 v84, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v27, 8, v28
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v24, 8, v25
+; ALIGNED-NEXT: v_lshl_or_b32 v67, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(23)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v22, 8, v26
+; ALIGNED-NEXT: s_waitcnt vmcnt(21)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v20, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v50, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(15)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v22
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v14
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v12, 8, v14
-; ALIGNED-NEXT: v_lshl_or_b32 v38, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v27, 8, v29
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v26, 8, v28
-; ALIGNED-NEXT: v_lshl_or_b32 v33, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v12, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v37, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v18, 8, v19
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v16, 8, v17
+; ALIGNED-NEXT: v_lshl_or_b32 v32, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v9, 8, v11
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v10
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v25, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v23, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v8
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v126, 8, v1
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v126
; ALIGNED-NEXT: v_lshl_or_b32 v15, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v108
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v94, 8, v106
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v124
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v110, 8, v120
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v90, 8, v92
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v78, 8, v88
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v104, 8, v106
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v91, 8, v94
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v121
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v108, 8, v110
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v89, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v75, 8, v78
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v74, 8, v76
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 16, v4
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v44, 8, v43
-; ALIGNED-NEXT: v_lshl_or_b32 v116, v87, 16, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v59, 8, v62
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v46, 8, v57
-; ALIGNED-NEXT: v_lshl_or_b32 v114, v87, 16, v86
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:1792
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v125, 8, v109
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v121, 8, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v111, 8, v127
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v97, 8, v113
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v56, 8, v60
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v58, 8, v73
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v119, 8, v102
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v117, 8, v40
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v73, 8, v76
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v85, 16, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v44, 8, v43
+; ALIGNED-NEXT: v_lshl_or_b32 v119, v86, 16, v85
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v59, 8, v58
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v46, 8, v56
+; ALIGNED-NEXT: v_lshl_or_b32 v116, v86, 16, v85
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:1792
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v102, 8, v107
+; ALIGNED-NEXT: s_clause 0x2
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:1800
+; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:1801
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:1802
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v109, 8, v85
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v86, 16, v85
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v123, 8, v111
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v87, 8, v105
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v86, 16, v85
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v57, 8, v47
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v74, 8, v77
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v86, 16, v85
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v112, 8, v102
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v103, 8, v40
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v86, 16, v85
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x2
; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:1810
-; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1808
-; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:1809
-; ALIGNED-NEXT: buffer_store_dword v114, off, s[0:3], s32 offset:488
-; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:1808
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:1809
+; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT: buffer_store_dword v119, off, s[0:3], s32 offset:492
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:480
-; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v103, 3
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v104, vcc_lo
-; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:247
+; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v114, 3
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v115, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:247
; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:248
; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:246
-; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:252
-; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:250
; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:251
+; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:252
; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:249
-; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:245
-; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:239
-; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:238
-; ALIGNED-NEXT: flat_store_byte v[3:4], v61 offset:244
-; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:242
-; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:243
+; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:250
+; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:245
+; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:239
+; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:238
+; ALIGNED-NEXT: flat_store_byte v[3:4], v61 offset:243
+; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:244
; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:241
+; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:242
; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:237
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:504
-; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:508
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:508
; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:500
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:496
-; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:231
-; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:232
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:231
+; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:232
; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:230
-; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:236
-; ALIGNED-NEXT: flat_store_byte v[3:4], v105 offset:234
-; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:235
-; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:233
-; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:229
-; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:223
+; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:235
+; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:236
+; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:233
+; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:234
+; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:229
+; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:223
; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:224
-; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:222
-; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:228
+; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:222
+; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:227
+; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:228
+; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:225
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:226
-; ALIGNED-NEXT: flat_store_byte v[3:4], v1 offset:227
-; ALIGNED-NEXT: flat_store_byte v[3:4], v10 offset:225
-; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:221
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:448
-; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:460
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:456
-; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:452
-; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:206
-; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:208
-; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:207
-; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:210
-; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:212
-; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:211
-; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:209
-; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:215
+; ALIGNED-NEXT: flat_store_byte v[3:4], v10 offset:221
+; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:456
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:460
+; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:215
; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:216
-; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:219
; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:220
+; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:217
; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:218
-; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:219
-; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:217
-; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:213
-; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:205
-; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:472
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:476
-; ALIGNED-NEXT: buffer_store_dword v77, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v25 offset:207
+; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:206
+; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:211
+; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:209
+; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:210
+; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:205
+; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:472
+; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:464
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v2
-; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v112, v112, 8, v86
-; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v123, v87, 8, v97
-; ALIGNED-NEXT: v_lshl_or_b32 v112, v112, 16, v123
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[3:4], v35 offset:199
-; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:200
-; ALIGNED-NEXT: flat_store_byte v[3:4], v37 offset:198
+; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:200
+; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:198
+; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:203
; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:204
+; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:201
; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:202
-; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:203
-; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:201
-; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:197
-; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:191
+; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:197
+; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:191
; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:192
-; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:190
+; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:190
+; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:195
; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:196
+; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:193
; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:194
-; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:195
-; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:193
-; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:189
+; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:189
+; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v2
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: v_lshl_or_b32 v113, v113, 8, v86
+; ALIGNED-NEXT: s_waitcnt vmcnt(1)
+; ALIGNED-NEXT: v_lshl_or_b32 v127, v85, 8, v87
+; ALIGNED-NEXT: v_lshl_or_b32 v113, v113, 16, v127
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:552
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:556
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:548
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:183
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:183
; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:184
-; ALIGNED-NEXT: flat_store_byte v[3:4], v80 offset:182
-; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:188
-; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:186
-; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:187
-; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:185
-; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:181
-; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:175
-; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:176
-; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:182
+; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:187
+; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:188
+; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:185
+; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:186
+; ALIGNED-NEXT: flat_store_byte v[3:4], v80 offset:181
+; ALIGNED-NEXT: flat_store_byte v[3:4], v97 offset:175
+; ALIGNED-NEXT: flat_store_byte v[3:4], v96 offset:176
+; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:179
; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:180
+; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:177
; ALIGNED-NEXT: flat_store_byte v[3:4], v83 offset:178
-; ALIGNED-NEXT: flat_store_byte v[3:4], v84 offset:179
-; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:177
-; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:173
+; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:572
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:560
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:167
-; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:168
-; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:166
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:167
+; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:168
+; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:166
+; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:171
; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:172
-; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:170
-; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:171
-; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:169
-; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:165
+; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:169
+; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:170
+; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:165
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:159
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:160
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:158
-; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:164
-; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:162
-; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:163
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:163
+; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:164
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:161
+; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:162
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:157
@@ -15239,16 +15245,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:150
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:155
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:156
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:154
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:155
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:153
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:153
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:154
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:149
@@ -15263,16 +15269,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:142
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:147
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:148
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:146
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:147
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:145
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:145
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:146
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:141
@@ -15299,16 +15305,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:134
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:140
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:138
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:139
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:137
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:137
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:138
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:133
@@ -15323,19 +15329,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:126
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:132
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:130
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:131
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:130
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:128
+; ALIGNED-NEXT: flat_store_byte v[114:115], v0 offset:128
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616
@@ -15359,16 +15365,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:118
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:124
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:122
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:123
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:121
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:121
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:122
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:117
@@ -15383,16 +15389,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:110
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:116
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:114
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:115
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:113
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:113
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:114
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:109
@@ -15419,16 +15425,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:102
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:108
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:106
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:107
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:105
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:105
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:106
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:101
@@ -15443,16 +15449,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:94
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:100
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:98
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:99
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:97
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:97
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:98
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:93
@@ -15462,7 +15468,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:580
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
@@ -15479,16 +15485,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:86
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:92
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:90
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:91
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:89
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:89
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:90
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:85
@@ -15503,16 +15509,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:78
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:84
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:82
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:83
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:81
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:81
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:82
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:77
@@ -15539,16 +15545,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:70
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:75
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:76
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:74
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:75
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:73
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:73
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:74
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:69
@@ -15563,19 +15569,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:62
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:67
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:68
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:66
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:67
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:66
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:64
+; ALIGNED-NEXT: flat_store_byte v[114:115], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680
@@ -15585,13 +15591,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
@@ -15600,48 +15603,51 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:54
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:60
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:57
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:57
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:46
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:52
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:52
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:49
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:700
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:700
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692
@@ -15650,52 +15656,52 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:688
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:39
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:31
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:36
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[114:115], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648
@@ -15705,7 +15711,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644
-; ALIGNED-NEXT: buffer_store_dword v112, off, s[0:3], s32 offset:640
+; ALIGNED-NEXT: buffer_store_dword v113, off, s[0:3], s32 offset:640
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:23
@@ -15717,44 +15723,44 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:22
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:28
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:26
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:27
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:25
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:25
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:26
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:21
; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:15
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:16
-; ALIGNED-NEXT: flat_store_byte v[3:4], v87 offset:14
+; ALIGNED-NEXT: flat_store_byte v[3:4], v85 offset:14
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:20
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:18
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:19
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[103:104], v97 offset:16
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:18
+; ALIGNED-NEXT: flat_store_byte v[114:115], v87 offset:16
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:668
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:660
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
@@ -15762,26 +15768,28 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:7
-; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:8
-; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:10
-; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:6
-; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:12
-; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:11
-; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:9
-; ALIGNED-NEXT: flat_store_byte v[103:104], v40 offset:8
-; ALIGNED-NEXT: flat_store_byte v[103:104], v109 offset:2
-; ALIGNED-NEXT: flat_store_byte v[103:104], v121 offset:1
-; ALIGNED-NEXT: flat_store_byte v[103:104], v113 offset:4
-; ALIGNED-NEXT: flat_store_byte v[3:4], v125
-; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:4
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[103:104], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0xffffff00, v103
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, -1, v104, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:8
+; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:6
+; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:11
+; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:12
+; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:9
+; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:10
+; ALIGNED-NEXT: flat_store_byte v[114:115], v40 offset:8
+; ALIGNED-NEXT: flat_store_byte v[114:115], v107 offset:2
+; ALIGNED-NEXT: flat_store_byte v[114:115], v109 offset:1
+; ALIGNED-NEXT: flat_store_byte v[114:115], v105 offset:4
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[114:115], v0
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
+; ALIGNED-NEXT: v_add_co_u32 v114, vcc_lo, 0xffffff00, v114
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v115, null, -1, v115, vcc_lo
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0
+; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:3
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:4
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:2
-; ALIGNED-NEXT: flat_store_byte v[3:4], v127 offset:3
; ALIGNED-NEXT: s_cbranch_scc0 .LBB9_5
; ALIGNED-NEXT: .LBB9_6: ; %Flow17
; ALIGNED-NEXT: s_or_b32 exec_lo, exec_lo, s6
@@ -16015,31 +16023,31 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:128
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:64
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:32
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:16
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:8
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:4
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:2
@@ -16048,10 +16056,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: v_add_co_u32 v2, vcc_lo, v2, 3
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; ALIGNED-NEXT: s_and_b32 vcc_lo, exec_lo, s6
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:247
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:248
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:246
@@ -16068,10 +16076,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:239
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:238
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:237
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:231
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:232
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:230
@@ -16088,10 +16096,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:223
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:222
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:221
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:215
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:216
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:214
@@ -16108,10 +16116,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:207
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:206
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:205
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:199
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:200
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:198
@@ -16128,10 +16136,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:191
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:190
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:189
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:183
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:184
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:182
@@ -16148,10 +16156,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:175
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:174
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:173
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:167
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:168
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:166
@@ -16168,10 +16176,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:159
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:158
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:157
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:151
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:152
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:150
@@ -16203,10 +16211,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:128
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:127
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:126
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:119
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:120
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:118
@@ -16223,10 +16231,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:111
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:110
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:109
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:103
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:104
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:102
@@ -16243,26 +16251,26 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:95
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:94
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:93
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:78
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:77
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:80
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:79
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:84
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:83
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:82
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:81
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:86
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:85
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:87
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:88
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:86
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:92
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:91
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:90
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:89
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:85
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:84
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:83
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:82
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:81
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:80
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:79
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:78
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:77
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:71
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:72
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:70
@@ -16278,10 +16286,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:64
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:63
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:62
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:55
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:56
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:54
@@ -16443,10 +16451,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
; ALIGNED-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:250
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:251
@@ -16464,10 +16472,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:242
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:241
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:240
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:234
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:235
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:233
@@ -16484,10 +16492,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:226
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:225
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:224
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:218
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:219
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:217
@@ -16505,10 +16513,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:209
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:208
; ALIGNED-NEXT: s_and_b32 vcc_lo, exec_lo, s6
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:202
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:203
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:201
@@ -16525,10 +16533,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:194
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:193
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:192
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:186
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:187
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:185
@@ -16545,10 +16553,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:178
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:177
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:176
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:170
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:171
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:169
@@ -16565,10 +16573,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:162
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:161
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:160
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:154
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:155
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:153
@@ -16585,10 +16593,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:146
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:145
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:144
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:138
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:139
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:137
@@ -16605,10 +16613,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:130
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:129
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:128
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:122
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:123
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:121
@@ -16625,10 +16633,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:114
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:113
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:112
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:106
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:107
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:105
@@ -16645,30 +16653,30 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:98
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:97
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:96
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:81
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:80
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:83
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:82
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:87
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:86
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:85
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:84
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:89
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:88
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:91
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:90
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:91
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:89
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:95
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:94
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:93
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:88
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:87
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:86
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:85
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:84
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:83
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:82
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:81
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:74
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:75
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:73
@@ -16685,10 +16693,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:66
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:65
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:64
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:58
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:59
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:57
@@ -16705,10 +16713,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:50
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:49
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:48
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:42
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:43
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:41
@@ -16725,10 +16733,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:34
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:33
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:32
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:26
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:27
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:25
@@ -16745,10 +16753,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:18
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:17
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:16
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:10
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:11
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:9
@@ -17024,22 +17032,22 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) #1 {
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:98
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:97
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:96
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:87
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:86
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:85
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:84
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:81
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:80
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:83
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:82
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:95
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:94
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:93
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:92
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:89
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:88
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:91
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:90
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:89
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:88
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:87
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:86
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:85
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:84
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:83
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:82
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:81
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:80
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:79
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:78
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:77
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1845bb56be388..4e2853dfa305c 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -506,8 +506,6 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
@@ -542,16 +540,16 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
-; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
-; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xf0
-; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xe0
-; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xd0
-; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0xc0
-; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
-; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
-; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0xf0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0xe0
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0xd0
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xc0
+; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xb0
+; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xa0
+; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0x90
+; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x50
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
; GFX942-SDAG-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -561,32 +559,32 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[38:39], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[20:23] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, 48
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[12:15] offset:32
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[8:11] offset:16
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[60:61], v[38:39], 0, s[20:21]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[14:15]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7] offset:128
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[20:21]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[38:39], 0, 48
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[32:35]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[28:31]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[32:35]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[28:31]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[20:23]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[12:15]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[60:61], v[8:11]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[20:23]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[16:19]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[12:15]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[8:11]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[32:35]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[28:31]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[24:27]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[16:19]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB4_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -600,8 +598,6 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
; GFX942-SDAG-NEXT: flat_store_short v[0:1], v3 offset:1052
; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 offset:1054
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
@@ -697,8 +693,6 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
@@ -733,16 +727,16 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
-; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
-; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xf0
-; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xe0
-; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xd0
-; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0xc0
-; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
-; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
-; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0xf0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0xe0
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0xd0
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xc0
+; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xb0
+; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xa0
+; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0x90
+; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x50
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
; GFX942-SDAG-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -752,36 +746,34 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[36:37], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[18:21] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, 48
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[10:13] offset:32
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[6:9] offset:16
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[36:37], 0, s[20:21]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[14:15]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5] offset:128
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[20:21]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[36:37], 0, 48
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[26:29]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[10:13]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[6:9]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[18:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[14:17]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[10:13]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[6:9]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[30:33]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[22:25]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[14:17]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB5_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index fc1c2d7d16d21..0702a03087b6f 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -86,59 +86,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
@@ -228,59 +228,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
@@ -347,22 +347,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
@@ -377,10 +377,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr
; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
@@ -402,7 +402,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX908-NEXT: v_mov_b32_e32 v32, 0
-; GFX908-NEXT: ; implicit-def: $agpr100_agpr101_agpr102_agpr103_agpr104_agpr105_agpr106_agpr107_agpr108_agpr109_agpr110_agpr111_agpr112_agpr113_agpr114_agpr115_agpr116_agpr117_agpr118_agpr119_agpr120_agpr121_agpr122_agpr123_agpr124_agpr125_agpr126_agpr127_agpr128_agpr129_agpr130_agpr131
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a[100:131]
; GFX908-NEXT: ;;#ASMEND
@@ -454,22 +453,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
@@ -484,10 +483,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add
; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
@@ -560,22 +559,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
@@ -590,10 +589,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr
; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
@@ -639,7 +638,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: s_mov_b64 s[2:3], s[38:39]
; GFX908-NEXT: s_mov_b32 s32, 0
; GFX908-NEXT: v_mov_b32_e32 v40, 0
-; GFX908-NEXT: ; implicit-def: $sgpr15
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_load_dwordx4 v[28:31], v40, s[34:35] offset:112
@@ -689,22 +687,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
@@ -719,10 +717,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:112
; GFX908-NEXT: global_store_dwordx4 v40, v[8:11], s[34:35] offset:64
; GFX908-NEXT: global_store_dwordx4 v40, v[12:15], s[34:35] offset:80
@@ -833,45 +831,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v6, v3, a[0:31] cbsz:1 abid:2 blgp:3
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a25
; GFX908-NEXT: v_accvgpr_read_b32 v3, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a27
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
; GFX908-NEXT: v_accvgpr_read_b32 v3, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:112
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a17
; GFX908-NEXT: v_accvgpr_read_b32 v3, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a19
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:64
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a21
; GFX908-NEXT: v_accvgpr_read_b32 v3, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a23
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:80
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a9
; GFX908-NEXT: v_accvgpr_read_b32 v3, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a11
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:32
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a13
; GFX908-NEXT: v_accvgpr_read_b32 v3, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:48
; GFX908-NEXT: s_nop 0
@@ -882,10 +880,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a5
; GFX908-NEXT: v_accvgpr_read_b32 v3, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a7
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:16
; GFX908-NEXT: s_cbranch_scc1 .LBB6_2
@@ -975,22 +973,22 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
; GFX908-NEXT: v_accvgpr_read_b32 v2, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
; GFX908-NEXT: v_accvgpr_read_b32 v6, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
; GFX908-NEXT: v_accvgpr_read_b32 v10, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
; GFX908-NEXT: v_accvgpr_read_b32 v14, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
; GFX908-NEXT: v_accvgpr_read_b32 v21, a11
; GFX908-NEXT: v_accvgpr_read_b32 v20, a10
; GFX908-NEXT: v_accvgpr_read_b32 v19, a9
@@ -1005,10 +1003,10 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0
; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
@@ -1076,22 +1074,22 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
; GFX908-NEXT: v_accvgpr_read_b32 v2, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
; GFX908-NEXT: v_accvgpr_read_b32 v6, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
; GFX908-NEXT: v_accvgpr_read_b32 v10, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
; GFX908-NEXT: v_accvgpr_read_b32 v14, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
; GFX908-NEXT: v_accvgpr_read_b32 v21, a11
; GFX908-NEXT: v_accvgpr_read_b32 v20, a10
; GFX908-NEXT: v_accvgpr_read_b32 v19, a9
@@ -1106,10 +1104,10 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..cfa0cf849d44d 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -159,59 +159,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; GREEDY908-NEXT: s_nop 15
; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a27
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a26
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a25
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a24
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a25
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a26
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a27
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a31
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a30
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a29
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a28
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a29
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a30
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a31
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a17
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a16
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a17
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a23
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a22
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a21
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a20
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a21
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a22
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a23
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a8
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a12
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a0
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a4
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; GREEDY908-NEXT: s_endpgm
@@ -627,31 +627,31 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-NEXT: s_nop 0
; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; GREEDY908-NEXT: s_nop 9
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a12
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17] offset:48
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a8
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17] offset:32
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a4
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17] offset:16
; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a0
+; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GREEDY908-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll b/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll
index cdf1709c3cbca..e842d06cbf118 100644
--- a/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll
@@ -10,74 +10,73 @@ define protected amdgpu_kernel void @no_folding_imm_to_inst_with_fi(<4 x i64> %v
; CHECK-NEXT: s_load_b512 s[0:15], s[4:5], 0xa4
; CHECK-NEXT: s_mov_b64 s[34:35], src_private_base
; CHECK-NEXT: s_movk_i32 s34, 0x80
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT: v_dual_mov_b32 v27, s35 :: v_dual_mov_b32 v26, s34
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT: v_dual_mov_b32 v25, s35 :: v_dual_mov_b32 v24, s34
; CHECK-NEXT: s_add_nc_u64 s[44:45], s[34:35], 0x70
-; CHECK-NEXT: v_dual_mov_b32 v20, s44 :: v_dual_mov_b32 v21, s45
+; CHECK-NEXT: s_add_nc_u64 s[46:47], s[34:35], 0x60
+; CHECK-NEXT: s_add_nc_u64 s[48:49], s[34:35], 0x50
+; CHECK-NEXT: v_dual_mov_b32 v26, s44 :: v_dual_mov_b32 v27, s45
+; CHECK-NEXT: s_add_nc_u64 s[34:35], s[34:35], 48
+; CHECK-NEXT: v_dual_mov_b32 v28, s46 :: v_dual_mov_b32 v29, s47
+; CHECK-NEXT: v_dual_mov_b32 v30, s48 :: v_dual_mov_b32 v31, s49
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v1, s41
; CHECK-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43
; CHECK-NEXT: v_dual_mov_b32 v4, s36 :: v_dual_mov_b32 v5, s37
; CHECK-NEXT: v_dual_mov_b32 v6, s38 :: v_dual_mov_b32 v7, s39
-; CHECK-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
-; CHECK-NEXT: s_add_nc_u64 s[24:25], s[34:35], 0x60
; CHECK-NEXT: v_dual_mov_b32 v8, s28 :: v_dual_mov_b32 v9, s29
; CHECK-NEXT: v_dual_mov_b32 v10, s30 :: v_dual_mov_b32 v11, s31
-; CHECK-NEXT: v_dual_mov_b32 v16, s20 :: v_dual_mov_b32 v17, s21
-; CHECK-NEXT: s_add_nc_u64 s[20:21], s[34:35], 0x50
+; CHECK-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; CHECK-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
-; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
-; CHECK-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v23, s25
+; CHECK-NEXT: v_dual_mov_b32 v16, s20 :: v_dual_mov_b32 v17, s21
; CHECK-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v19, s23
-; CHECK-NEXT: v_dual_mov_b32 v25, s21 :: v_dual_mov_b32 v24, s20
+; CHECK-NEXT: v_dual_mov_b32 v23, s19 :: v_dual_mov_b32 v22, s18
+; CHECK-NEXT: v_dual_mov_b32 v21, s17 :: v_dual_mov_b32 v20, s16
; CHECK-NEXT: scratch_store_b128 off, v[0:3], off offset:16 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
; CHECK-NEXT: scratch_store_b128 off, v[4:7], off scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[20:21], v[8:11] scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[26:27], v[8:11] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[22:23], v[12:15] scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[28:29], v[12:15] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[24:25], v[16:19] scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[30:31], v[16:19] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
-; CHECK-NEXT: s_add_nc_u64 s[12:13], s[34:35], 48
-; CHECK-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
-; CHECK-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v3, s19
-; CHECK-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
-; CHECK-NEXT: v_dual_mov_b32 v29, s13 :: v_dual_mov_b32 v28, s12
-; CHECK-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; CHECK-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; CHECK-NEXT: v_dual_mov_b32 v12, s4 :: v_dual_mov_b32 v13, s5
-; CHECK-NEXT: v_dual_mov_b32 v14, s6 :: v_dual_mov_b32 v15, s7
-; CHECK-NEXT: v_dual_mov_b32 v16, s0 :: v_dual_mov_b32 v17, s1
-; CHECK-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s3
-; CHECK-NEXT: flat_store_b128 v[26:27], v[0:3] offset:64 scope:SCOPE_SYS
+; CHECK-NEXT: v_dual_mov_b32 v16, s34 :: v_dual_mov_b32 v17, s35
+; CHECK-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
+; CHECK-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
+; CHECK-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v5, s9
+; CHECK-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v7, s11
+; CHECK-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
+; CHECK-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v11, s7
+; CHECK-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v14, s2
+; CHECK-NEXT: v_dual_mov_b32 v13, s1 :: v_dual_mov_b32 v12, s0
+; CHECK-NEXT: flat_store_b128 v[24:25], v[20:23] offset:64 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[28:29], v[4:7] scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[16:17], v[0:3] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[26:27], v[8:11] offset:32 scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[24:25], v[4:7] offset:32 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[26:27], v[12:15] offset:16 scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[24:25], v[8:11] offset:16 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_store_b128 v[26:27], v[16:19] scope:SCOPE_SYS
+; CHECK-NEXT: flat_store_b128 v[24:25], v[12:15] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[22:23] scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[28:29] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[20:21] scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[26:27] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[26:27] offset:64 scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[24:25] offset:64 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[24:25] scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[30:31] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[26:27] offset:32 scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[24:25] offset:32 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[28:29] scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[16:17] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[26:27] scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[24:25] scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: flat_load_b128 v[0:3], v[26:27] offset:16 scope:SCOPE_SYS
+; CHECK-NEXT: flat_load_b128 v[0:3], v[24:25] offset:16 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_loadcnt 0x0
; CHECK-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 4afa95c836be9..2bfb11d4f868e 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -156,10 +156,10 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
; GFX1250-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[4:5]
; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[6:7]
; GFX1250-SDAG-NEXT: global_store_b128 v8, v[0:3], s[6:7] scale_offset
@@ -1640,10 +1640,10 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
; GFX1250-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[4:5]
; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
; GFX1250-SDAG-NEXT: global_store_b128 v8, v[0:3], s[6:7] scale_offset
@@ -4367,20 +4367,44 @@ define <4 x float> @fadd_v4f32_neg1(<4 x float> %first, <4 x float> %second) {
; GFX900-NEXT: v_sub_f32_e32 v3, v3, v7
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; PACKED-LABEL: fadd_v4f32_neg1:
-; PACKED: ; %bb.0:
-; PACKED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; PACKED-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: s_setpc_b64 s[30:31]
+; PACKED-SDAG-LABEL: fadd_v4f32_neg1:
+; PACKED-SDAG: ; %bb.0:
+; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v7
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v6
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[4:5]
+; PACKED-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: fadd_v4f32_neg1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; PACKED-GISEL-LABEL: fadd_v4f32_neg1:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: fadd_v4f32_neg1:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v7, 0x80000000, v7
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v6, 0x80000000, v6
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7]
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: fadd_v4f32_neg1:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
%neg = fneg <4 x float> %second
%add = fadd <4 x float> %first, %neg
ret <4 x float> %add
@@ -4400,24 +4424,60 @@ define <8 x float> @fadd_v8f32_neg0(<8 x float> %first, <8 x float> %second) {
; GFX900-NEXT: v_sub_f32_e32 v7, v15, v7
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; PACKED-LABEL: fadd_v8f32_neg0:
-; PACKED: ; %bb.0:
-; PACKED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; PACKED-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: s_setpc_b64 s[30:31]
+; PACKED-SDAG-LABEL: fadd_v8f32_neg0:
+; PACKED-SDAG: ; %bb.0:
+; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v7, 0x80000000, v7
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v6, 0x80000000, v6
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7]
+; PACKED-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: fadd_v8f32_neg0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; PACKED-GISEL-LABEL: fadd_v8f32_neg0:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: fadd_v8f32_neg0:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v7, 0x80000000, v7
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v6, 0x80000000, v6
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7]
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: fadd_v8f32_neg0:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[8:9], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[2:3], v[10:11], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[4:5], v[12:13], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[6:7], v[14:15], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
%neg = fneg <8 x float> %first
%add = fadd <8 x float> %neg, %second
ret <8 x float> %add
@@ -4457,20 +4517,44 @@ define <4 x float> @fsub_v4f32(<4 x float> %first, <4 x float> %second) {
; GFX900-NEXT: v_sub_f32_e32 v3, v3, v7
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; PACKED-LABEL: fsub_v4f32:
-; PACKED: ; %bb.0:
-; PACKED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; PACKED-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: s_setpc_b64 s[30:31]
+; PACKED-SDAG-LABEL: fsub_v4f32:
+; PACKED-SDAG: ; %bb.0:
+; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v7
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v6
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[4:5]
+; PACKED-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: fsub_v4f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; PACKED-GISEL-LABEL: fsub_v4f32:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: fsub_v4f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v7, 0x80000000, v7
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v6, 0x80000000, v6
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7]
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: fsub_v4f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
%sub = fsub <4 x float> %first, %second
ret <4 x float> %sub
}
@@ -4489,24 +4573,60 @@ define <8 x float> @fsub_v8f32(<8 x float> %first, <8 x float> %second) {
; GFX900-NEXT: v_sub_f32_e32 v7, v7, v15
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; PACKED-LABEL: fsub_v8f32:
-; PACKED: ; %bb.0:
-; PACKED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; PACKED-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[12:13] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[14:15] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-NEXT: s_setpc_b64 s[30:31]
+; PACKED-SDAG-LABEL: fsub_v8f32:
+; PACKED-SDAG: ; %bb.0:
+; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v9, 0x80000000, v9
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v8, 0x80000000, v8
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9]
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v9, 0x80000000, v11
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v8, 0x80000000, v10
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[8:9]
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v9, 0x80000000, v13
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v8, 0x80000000, v12
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[8:9]
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v9, 0x80000000, v15
+; PACKED-SDAG-NEXT: v_xor_b32_e32 v8, 0x80000000, v14
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[8:9]
+; PACKED-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: fsub_v8f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[12:13] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[14:15] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; PACKED-GISEL-LABEL: fsub_v8f32:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[12:13] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[14:15] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: fsub_v8f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v9, 0x80000000, v9
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v8, 0x80000000, v8
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v11, 0x80000000, v11
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v10, 0x80000000, v10
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v13, 0x80000000, v13
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v12, 0x80000000, v12
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v15, 0x80000000, v15
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v14, 0x80000000, v14
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[12:13]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[14:15]
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: fsub_v8f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[12:13] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[14:15] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
%sub = fsub <8 x float> %first, %second
ret <8 x float> %sub
}
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index 5e948144960f1..6bd6afdf25c47 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -103,61 +103,33 @@ define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
}
define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
-; GFX1251-SDAG-LABEL: fadd_v4_vs:
-; GFX1251-SDAG: ; %bb.0:
-; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-SDAG-NEXT: v_nop
-; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[12:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[4:7], v[4:7], v[8:11]
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-SDAG-NEXT: s_endpgm
-;
-; GFX1251-GISEL-LABEL: fadd_v4_vs:
-; GFX1251-GISEL: ; %bb.0:
-; GFX1251-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-GISEL-NEXT: v_nop
-; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-GISEL-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-GISEL-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[8:11]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1251-GISEL-NEXT: v_pk_add_f64 v[4:7], v[4:7], v[12:15]
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-GISEL-NEXT: s_endpgm
+; GFX1251-LABEL: fadd_v4_vs:
+; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: v_nop
+; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-NEXT: v_lshlrev_b32_e32 v16, 5, v0
+; GFX1251-NEXT: s_wait_kmcnt 0x0
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[0:3], v16, s[0:1]
+; GFX1251-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
+; GFX1251-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX1251-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX1251-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX1251-NEXT: s_wait_loadcnt 0x1
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1251-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[8:11]
+; GFX1251-NEXT: s_wait_loadcnt 0x0
+; GFX1251-NEXT: v_pk_add_f64 v[4:7], v[4:7], v[12:15]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX1251-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX1251-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x double>, ptr addrspace(1) %a, i32 %id
%load = load <4 x double>, ptr addrspace(1) %gep, align 16
@@ -1005,61 +977,33 @@ define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
}
define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
-; GFX1251-SDAG-LABEL: fmul_v4_vs:
-; GFX1251-SDAG: ; %bb.0:
-; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-SDAG-NEXT: v_nop
-; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[0:3], v[12:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[4:7], v[4:7], v[8:11]
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-SDAG-NEXT: s_endpgm
-;
-; GFX1251-GISEL-LABEL: fmul_v4_vs:
-; GFX1251-GISEL: ; %bb.0:
-; GFX1251-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-GISEL-NEXT: v_nop
-; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-GISEL-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-GISEL-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_pk_mul_f64 v[0:3], v[0:3], v[8:11]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1251-GISEL-NEXT: v_pk_mul_f64 v[4:7], v[4:7], v[12:15]
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-GISEL-NEXT: s_endpgm
+; GFX1251-LABEL: fmul_v4_vs:
+; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: v_nop
+; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-NEXT: v_lshlrev_b32_e32 v16, 5, v0
+; GFX1251-NEXT: s_wait_kmcnt 0x0
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[0:3], v16, s[0:1]
+; GFX1251-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
+; GFX1251-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX1251-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX1251-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX1251-NEXT: s_wait_loadcnt 0x1
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1251-NEXT: v_pk_mul_f64 v[0:3], v[0:3], v[8:11]
+; GFX1251-NEXT: s_wait_loadcnt 0x0
+; GFX1251-NEXT: v_pk_mul_f64 v[4:7], v[4:7], v[12:15]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX1251-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX1251-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x double>, ptr addrspace(1) %a, i32 %id
%load = load <4 x double>, ptr addrspace(1) %gep, align 16
@@ -1538,8 +1482,8 @@ define amdgpu_kernel void @fma_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
; GFX1251-SDAG-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX1251-SDAG-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
; GFX1251-SDAG-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
-; GFX1251-SDAG-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
-; GFX1251-SDAG-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, s3
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[4:7], v[8:11]
; GFX1251-SDAG-NEXT: v_mov_b32_e32 v4, 0
@@ -1640,8 +1584,8 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-NEXT: global_load_b128 v[16:19], v44, s[0:1] offset:240
; GFX1251-SDAG-NEXT: global_load_b128 v[12:15], v44, s[0:1] offset:224
; GFX1251-SDAG-NEXT: global_load_b128 v[8:11], v44, s[0:1] offset:16
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v44, s[0:1]
-; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v44, s[0:1] offset:80
+; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v44, s[0:1]
+; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v44, s[0:1] offset:80
; GFX1251-SDAG-NEXT: global_load_b128 v[46:49], v44, s[0:1] offset:64
; GFX1251-SDAG-NEXT: global_load_b128 v[50:53], v44, s[0:1] offset:112
; GFX1251-SDAG-NEXT: global_load_b128 v[54:57], v44, s[0:1] offset:96
@@ -1690,11 +1634,11 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[68:69], s[42:43]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[66:67], s[40:41]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x6
-; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[70:73], v[70:73]
+; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[4:7], v[4:7], v[70:73], v[70:73]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[72:73], s[38:39]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[70:71], s[36:37]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x5
-; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[4:7], v[4:7], v[66:69], v[66:69]
+; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[66:69], v[66:69]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[68:69], s[50:51]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[66:67], s[48:49]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x4
@@ -1718,10 +1662,10 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[54:57], s[0:1] offset:96
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[50:53], s[0:1] offset:112
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[46:49], s[0:1] offset:64
-; GFX1251-SDAG-NEXT: global_store_b128 v44, v[4:7], s[0:1] offset:80
+; GFX1251-SDAG-NEXT: global_store_b128 v44, v[0:3], s[0:1] offset:80
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[58:61], s[0:1] offset:32
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[62:65], s[0:1] offset:48
-; GFX1251-SDAG-NEXT: global_store_b128 v44, v[0:3], s[0:1]
+; GFX1251-SDAG-NEXT: global_store_b128 v44, v[4:7], s[0:1]
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[8:11], s[0:1] offset:16
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[12:15], s[0:1] offset:224
; GFX1251-SDAG-NEXT: global_store_b128 v44, v[16:19], s[0:1] offset:240
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index d551aad58b990..9fbae89fc0159 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -103,61 +103,33 @@ define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
}
define amdgpu_kernel void @add_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
-; GFX1251-SDAG-LABEL: add_v4_vs:
-; GFX1251-SDAG: ; %bb.0:
-; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-SDAG-NEXT: v_nop
-; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[12:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[4:7], v[4:7], v[8:11]
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-SDAG-NEXT: s_endpgm
-;
-; GFX1251-GISEL-LABEL: add_v4_vs:
-; GFX1251-GISEL: ; %bb.0:
-; GFX1251-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-GISEL-NEXT: v_nop
-; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-GISEL-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-GISEL-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[8:11]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1251-GISEL-NEXT: v_pk_add_nc_u64 v[4:7], v[4:7], v[12:15]
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-GISEL-NEXT: s_endpgm
+; GFX1251-LABEL: add_v4_vs:
+; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: v_nop
+; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-NEXT: v_lshlrev_b32_e32 v16, 5, v0
+; GFX1251-NEXT: s_wait_kmcnt 0x0
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[0:3], v16, s[0:1]
+; GFX1251-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
+; GFX1251-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX1251-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX1251-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX1251-NEXT: s_wait_loadcnt 0x1
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[8:11]
+; GFX1251-NEXT: s_wait_loadcnt 0x0
+; GFX1251-NEXT: v_pk_add_nc_u64 v[4:7], v[4:7], v[12:15]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX1251-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX1251-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x i64>, ptr addrspace(1) %a, i32 %id
%load = load <4 x i64>, ptr addrspace(1) %gep, align 16
@@ -757,61 +729,33 @@ define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
}
define amdgpu_kernel void @sub_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
-; GFX1251-SDAG-LABEL: sub_v4_vs:
-; GFX1251-SDAG: ; %bb.0:
-; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-SDAG-NEXT: v_nop
-; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-SDAG-NEXT: v_pk_sub_nc_u64 v[0:3], v[0:3], v[12:15]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1251-SDAG-NEXT: v_pk_sub_nc_u64 v[4:7], v[4:7], v[8:11]
-; GFX1251-SDAG-NEXT: s_clause 0x1
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-SDAG-NEXT: s_endpgm
-;
-; GFX1251-GISEL-LABEL: sub_v4_vs:
-; GFX1251-GISEL: ; %bb.0:
-; GFX1251-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1251-GISEL-NEXT: v_nop
-; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-GISEL-NEXT: v_lshlrev_b32_e32 v16, 5, v0
-; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_load_b128 v[0:3], v16, s[0:1]
-; GFX1251-GISEL-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_pk_sub_nc_u64 v[0:3], v[0:3], v[8:11]
-; GFX1251-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1251-GISEL-NEXT: v_pk_sub_nc_u64 v[4:7], v[4:7], v[12:15]
-; GFX1251-GISEL-NEXT: s_clause 0x1
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
-; GFX1251-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
-; GFX1251-GISEL-NEXT: s_endpgm
+; GFX1251-LABEL: sub_v4_vs:
+; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: v_nop
+; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-NEXT: v_lshlrev_b32_e32 v16, 5, v0
+; GFX1251-NEXT: s_wait_kmcnt 0x0
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[0:3], v16, s[0:1]
+; GFX1251-NEXT: global_load_b128 v[4:7], v16, s[0:1] offset:16
+; GFX1251-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX1251-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX1251-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX1251-NEXT: s_wait_loadcnt 0x1
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1251-NEXT: v_pk_sub_nc_u64 v[0:3], v[0:3], v[8:11]
+; GFX1251-NEXT: s_wait_loadcnt 0x0
+; GFX1251-NEXT: v_pk_sub_nc_u64 v[4:7], v[4:7], v[12:15]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX1251-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX1251-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x i64>, ptr addrspace(1) %a, i32 %id
%load = load <4 x i64>, ptr addrspace(1) %gep, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index 526e9d66bf0b0..1f8ad717871d4 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -108,12 +108,12 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s(<2 x i64> %v, <2 x i64> %a) {
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[8:9], 2
; GFX1251-NEXT: s_lshl_b64 s[2:3], s[10:11], 2
-; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[4:7], v[0:3]
+; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[4:7]
; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX1251-NEXT: s_endpgm
%shl = shl <2 x i64> %v, <i64 2, i64 2>
@@ -185,12 +185,12 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_3(<2 x i64> %v, <2 x i64>
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[8:9], 2
; GFX1251-NEXT: s_lshl_b64 s[2:3], s[10:11], 3
-; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[4:7], v[0:3]
+; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[4:7]
; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX1251-NEXT: s_endpgm
%shl = shl <2 x i64> %v, <i64 2, i64 3>
@@ -211,12 +211,12 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_4(<2 x i64> %v, <2 x i64>
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[8:9], 2
; GFX1251-NEXT: s_lshl_b64 s[2:3], s[10:11], 4
-; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[4:7], v[0:3]
+; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[4:7]
; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX1251-NEXT: s_endpgm
%shl = shl <2 x i64> %v, <i64 2, i64 4>
@@ -235,12 +235,12 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_5(<2 x i64> %v, <2 x i64>
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[8:9], 2
; GFX1251-NEXT: s_lshl_b64 s[2:3], s[10:11], 5
-; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1251-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[4:7], v[0:3]
+; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[4:7]
; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX1251-NEXT: s_endpgm
%shl = shl <2 x i64> %v, <i64 2, i64 5>
diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index 0d817f6497f1a..98952085f16a9 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -459,19 +459,14 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: .LBB9_0:
; GFX942-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: v_mov_b32_e32 v1, s9
-; GFX942-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v0, s4
-; GFX942-NEXT: v_mov_b32_e32 v1, s5
-; GFX942-NEXT: v_mov_b32_e32 v2, s6
-; GFX942-NEXT: v_mov_b32_e32 v3, s7
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[2:3] offset:16
+; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[2:3]
; GFX942-NEXT: s_endpgm
;
; GFX90a-LABEL: v8i32_arg:
@@ -483,19 +478,14 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x
; GFX90a-NEXT: ; %bb.2:
; GFX90a-NEXT: .LBB9_0:
; GFX90a-NEXT: s_load_dwordx8 s[12:19], s[4:5], 0x20
-; GFX90a-NEXT: v_mov_b32_e32 v4, 0
+; GFX90a-NEXT: v_mov_b32_e32 v0, 0
; GFX90a-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT: v_mov_b32_e32 v0, s16
-; GFX90a-NEXT: v_mov_b32_e32 v1, s17
-; GFX90a-NEXT: v_mov_b32_e32 v2, s18
-; GFX90a-NEXT: v_mov_b32_e32 v3, s19
-; GFX90a-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
-; GFX90a-NEXT: s_nop 0
-; GFX90a-NEXT: v_mov_b32_e32 v0, s12
-; GFX90a-NEXT: v_mov_b32_e32 v1, s13
-; GFX90a-NEXT: v_mov_b32_e32 v2, s14
-; GFX90a-NEXT: v_mov_b32_e32 v3, s15
-; GFX90a-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
+; GFX90a-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90a-NEXT: v_pk_mov_b32 v[4:5], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90a-NEXT: v_pk_mov_b32 v[6:7], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90a-NEXT: v_pk_mov_b32 v[8:9], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90a-NEXT: global_store_dwordx4 v0, v[2:5], s[8:9] offset:16
+; GFX90a-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX90a-NEXT: s_endpgm
;
; GFX1250-LABEL: v8i32_arg:
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 79cd8d6fb05d2..77370afa40da6 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -46,28 +46,19 @@ define amdgpu_kernel void @store_v16i32(ptr addrspace(1) %out, <16 x i32> %a) {
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s20
-; GFX942-NEXT: v_mov_b32_e32 v3, s21
-; GFX942-NEXT: v_mov_b32_e32 v4, s22
-; GFX942-NEXT: v_mov_b32_e32 v5, s23
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[20:21]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[22:23]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[16:17]
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[18:19]
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48
-; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: v_mov_b32_e32 v3, s17
-; GFX942-NEXT: v_mov_b32_e32 v4, s18
-; GFX942-NEXT: v_mov_b32_e32 v5, s19
-; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32
-; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v2, s12
-; GFX942-NEXT: v_mov_b32_e32 v3, s13
-; GFX942-NEXT: v_mov_b32_e32 v4, s14
-; GFX942-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:32
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v2, s8
-; GFX942-NEXT: v_mov_b32_e32 v3, s9
-; GFX942-NEXT: v_mov_b32_e32 v4, s10
-; GFX942-NEXT: v_mov_b32_e32 v5, s11
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-single-exit-copy.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-single-exit-copy.ll
index cfbb80c7696a0..92ab19bfc323e 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-single-exit-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-single-exit-copy.ll
@@ -26,230 +26,230 @@ define amdgpu_kernel void @single_exit_copy(
; CHECK-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x0
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x20
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: v_mov_b32_e32 v31, 0
-; CHECK-NEXT: v_mov_b32_e32 v30, 0
-; CHECK-NEXT: v_mov_b32_e32 v29, 0
-; CHECK-NEXT: v_mov_b32_e32 v28, 0
-; CHECK-NEXT: v_mov_b32_e32 v27, 0
-; CHECK-NEXT: v_mov_b32_e32 v26, 0
-; CHECK-NEXT: v_mov_b32_e32 v25, 0
-; CHECK-NEXT: v_mov_b32_e32 v24, 0
-; CHECK-NEXT: v_mov_b32_e32 v23, 0
-; CHECK-NEXT: v_mov_b32_e32 v22, 0
-; CHECK-NEXT: v_mov_b32_e32 v21, 0
-; CHECK-NEXT: v_mov_b32_e32 v20, 0
-; CHECK-NEXT: v_mov_b32_e32 v19, 0
-; CHECK-NEXT: v_mov_b32_e32 v18, 0
-; CHECK-NEXT: v_mov_b32_e32 v17, 0
-; CHECK-NEXT: v_mov_b32_e32 v16, 0
-; CHECK-NEXT: v_mov_b32_e32 v15, 0
-; CHECK-NEXT: v_mov_b32_e32 v14, 0
-; CHECK-NEXT: v_mov_b32_e32 v13, 0
-; CHECK-NEXT: v_mov_b32_e32 v12, 0
-; CHECK-NEXT: v_mov_b32_e32 v11, 0
-; CHECK-NEXT: v_mov_b32_e32 v10, 0
-; CHECK-NEXT: v_mov_b32_e32 v9, 0
-; CHECK-NEXT: v_mov_b32_e32 v8, 0
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b32_e32 v63, 0
-; CHECK-NEXT: v_mov_b32_e32 v62, 0
-; CHECK-NEXT: v_mov_b32_e32 v61, 0
-; CHECK-NEXT: v_mov_b32_e32 v60, 0
-; CHECK-NEXT: v_mov_b32_e32 v59, 0
-; CHECK-NEXT: v_mov_b32_e32 v58, 0
-; CHECK-NEXT: v_mov_b32_e32 v57, 0
-; CHECK-NEXT: v_mov_b32_e32 v56, 0
-; CHECK-NEXT: v_mov_b32_e32 v55, 0
-; CHECK-NEXT: v_mov_b32_e32 v54, 0
-; CHECK-NEXT: v_mov_b32_e32 v53, 0
-; CHECK-NEXT: v_mov_b32_e32 v52, 0
-; CHECK-NEXT: v_mov_b32_e32 v51, 0
-; CHECK-NEXT: v_mov_b32_e32 v50, 0
-; CHECK-NEXT: v_mov_b32_e32 v49, 0
-; CHECK-NEXT: v_mov_b32_e32 v48, 0
-; CHECK-NEXT: v_mov_b32_e32 v47, 0
-; CHECK-NEXT: v_mov_b32_e32 v46, 0
-; CHECK-NEXT: v_mov_b32_e32 v45, 0
-; CHECK-NEXT: v_mov_b32_e32 v44, 0
-; CHECK-NEXT: v_mov_b32_e32 v43, 0
-; CHECK-NEXT: v_mov_b32_e32 v42, 0
-; CHECK-NEXT: v_mov_b32_e32 v41, 0
-; CHECK-NEXT: v_mov_b32_e32 v40, 0
-; CHECK-NEXT: v_mov_b32_e32 v39, 0
-; CHECK-NEXT: v_mov_b32_e32 v38, 0
-; CHECK-NEXT: v_mov_b32_e32 v37, 0
-; CHECK-NEXT: v_mov_b32_e32 v36, 0
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v11, 0
+; CHECK-NEXT: v_mov_b32_e32 v10, 0
+; CHECK-NEXT: v_mov_b32_e32 v9, 0
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v15, 0
+; CHECK-NEXT: v_mov_b32_e32 v14, 0
+; CHECK-NEXT: v_mov_b32_e32 v13, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, 0
+; CHECK-NEXT: v_mov_b32_e32 v19, 0
+; CHECK-NEXT: v_mov_b32_e32 v18, 0
+; CHECK-NEXT: v_mov_b32_e32 v17, 0
+; CHECK-NEXT: v_mov_b32_e32 v16, 0
+; CHECK-NEXT: v_mov_b32_e32 v23, 0
+; CHECK-NEXT: v_mov_b32_e32 v22, 0
+; CHECK-NEXT: v_mov_b32_e32 v21, 0
+; CHECK-NEXT: v_mov_b32_e32 v20, 0
+; CHECK-NEXT: v_mov_b32_e32 v27, 0
+; CHECK-NEXT: v_mov_b32_e32 v26, 0
+; CHECK-NEXT: v_mov_b32_e32 v25, 0
+; CHECK-NEXT: v_mov_b32_e32 v24, 0
+; CHECK-NEXT: v_mov_b32_e32 v31, 0
+; CHECK-NEXT: v_mov_b32_e32 v30, 0
+; CHECK-NEXT: v_mov_b32_e32 v29, 0
+; CHECK-NEXT: v_mov_b32_e32 v28, 0
; CHECK-NEXT: v_mov_b32_e32 v35, 0
; CHECK-NEXT: v_mov_b32_e32 v34, 0
; CHECK-NEXT: v_mov_b32_e32 v33, 0
; CHECK-NEXT: v_mov_b32_e32 v32, 0
-; CHECK-NEXT: v_mov_b32_e32 v95, 0
-; CHECK-NEXT: v_mov_b32_e32 v94, 0
-; CHECK-NEXT: v_mov_b32_e32 v93, 0
-; CHECK-NEXT: v_mov_b32_e32 v92, 0
-; CHECK-NEXT: v_mov_b32_e32 v91, 0
-; CHECK-NEXT: v_mov_b32_e32 v90, 0
-; CHECK-NEXT: v_mov_b32_e32 v89, 0
-; CHECK-NEXT: v_mov_b32_e32 v88, 0
-; CHECK-NEXT: v_mov_b32_e32 v87, 0
-; CHECK-NEXT: v_mov_b32_e32 v86, 0
-; CHECK-NEXT: v_mov_b32_e32 v85, 0
-; CHECK-NEXT: v_mov_b32_e32 v84, 0
-; CHECK-NEXT: v_mov_b32_e32 v83, 0
-; CHECK-NEXT: v_mov_b32_e32 v82, 0
-; CHECK-NEXT: v_mov_b32_e32 v81, 0
-; CHECK-NEXT: v_mov_b32_e32 v80, 0
-; CHECK-NEXT: v_mov_b32_e32 v79, 0
-; CHECK-NEXT: v_mov_b32_e32 v78, 0
-; CHECK-NEXT: v_mov_b32_e32 v77, 0
-; CHECK-NEXT: v_mov_b32_e32 v76, 0
-; CHECK-NEXT: v_mov_b32_e32 v75, 0
-; CHECK-NEXT: v_mov_b32_e32 v74, 0
-; CHECK-NEXT: v_mov_b32_e32 v73, 0
-; CHECK-NEXT: v_mov_b32_e32 v72, 0
-; CHECK-NEXT: v_mov_b32_e32 v71, 0
-; CHECK-NEXT: v_mov_b32_e32 v70, 0
-; CHECK-NEXT: v_mov_b32_e32 v69, 0
-; CHECK-NEXT: v_mov_b32_e32 v68, 0
+; CHECK-NEXT: v_mov_b32_e32 v39, 0
+; CHECK-NEXT: v_mov_b32_e32 v38, 0
+; CHECK-NEXT: v_mov_b32_e32 v37, 0
+; CHECK-NEXT: v_mov_b32_e32 v36, 0
+; CHECK-NEXT: v_mov_b32_e32 v43, 0
+; CHECK-NEXT: v_mov_b32_e32 v42, 0
+; CHECK-NEXT: v_mov_b32_e32 v41, 0
+; CHECK-NEXT: v_mov_b32_e32 v40, 0
+; CHECK-NEXT: v_mov_b32_e32 v47, 0
+; CHECK-NEXT: v_mov_b32_e32 v46, 0
+; CHECK-NEXT: v_mov_b32_e32 v45, 0
+; CHECK-NEXT: v_mov_b32_e32 v44, 0
+; CHECK-NEXT: v_mov_b32_e32 v51, 0
+; CHECK-NEXT: v_mov_b32_e32 v50, 0
+; CHECK-NEXT: v_mov_b32_e32 v49, 0
+; CHECK-NEXT: v_mov_b32_e32 v48, 0
+; CHECK-NEXT: v_mov_b32_e32 v55, 0
+; CHECK-NEXT: v_mov_b32_e32 v54, 0
+; CHECK-NEXT: v_mov_b32_e32 v53, 0
+; CHECK-NEXT: v_mov_b32_e32 v52, 0
+; CHECK-NEXT: v_mov_b32_e32 v59, 0
+; CHECK-NEXT: v_mov_b32_e32 v58, 0
+; CHECK-NEXT: v_mov_b32_e32 v57, 0
+; CHECK-NEXT: v_mov_b32_e32 v56, 0
+; CHECK-NEXT: v_mov_b32_e32 v63, 0
+; CHECK-NEXT: v_mov_b32_e32 v62, 0
+; CHECK-NEXT: v_mov_b32_e32 v61, 0
+; CHECK-NEXT: v_mov_b32_e32 v60, 0
; CHECK-NEXT: v_mov_b32_e32 v67, 0
; CHECK-NEXT: v_mov_b32_e32 v66, 0
; CHECK-NEXT: v_mov_b32_e32 v65, 0
; CHECK-NEXT: v_mov_b32_e32 v64, 0
-; CHECK-NEXT: v_mov_b32_e32 v127, 0
-; CHECK-NEXT: v_mov_b32_e32 v126, 0
-; CHECK-NEXT: v_mov_b32_e32 v125, 0
-; CHECK-NEXT: v_mov_b32_e32 v124, 0
-; CHECK-NEXT: v_mov_b32_e32 v123, 0
-; CHECK-NEXT: v_mov_b32_e32 v122, 0
-; CHECK-NEXT: v_mov_b32_e32 v121, 0
-; CHECK-NEXT: v_mov_b32_e32 v120, 0
-; CHECK-NEXT: v_mov_b32_e32 v119, 0
-; CHECK-NEXT: v_mov_b32_e32 v118, 0
-; CHECK-NEXT: v_mov_b32_e32 v117, 0
-; CHECK-NEXT: v_mov_b32_e32 v116, 0
-; CHECK-NEXT: v_mov_b32_e32 v115, 0
-; CHECK-NEXT: v_mov_b32_e32 v114, 0
-; CHECK-NEXT: v_mov_b32_e32 v113, 0
-; CHECK-NEXT: v_mov_b32_e32 v112, 0
-; CHECK-NEXT: v_mov_b32_e32 v111, 0
-; CHECK-NEXT: v_mov_b32_e32 v110, 0
-; CHECK-NEXT: v_mov_b32_e32 v109, 0
-; CHECK-NEXT: v_mov_b32_e32 v108, 0
-; CHECK-NEXT: v_mov_b32_e32 v107, 0
-; CHECK-NEXT: v_mov_b32_e32 v106, 0
-; CHECK-NEXT: v_mov_b32_e32 v105, 0
-; CHECK-NEXT: v_mov_b32_e32 v104, 0
-; CHECK-NEXT: v_mov_b32_e32 v103, 0
-; CHECK-NEXT: v_mov_b32_e32 v102, 0
-; CHECK-NEXT: v_mov_b32_e32 v101, 0
-; CHECK-NEXT: v_mov_b32_e32 v100, 0
+; CHECK-NEXT: v_mov_b32_e32 v71, 0
+; CHECK-NEXT: v_mov_b32_e32 v70, 0
+; CHECK-NEXT: v_mov_b32_e32 v69, 0
+; CHECK-NEXT: v_mov_b32_e32 v68, 0
+; CHECK-NEXT: v_mov_b32_e32 v75, 0
+; CHECK-NEXT: v_mov_b32_e32 v74, 0
+; CHECK-NEXT: v_mov_b32_e32 v73, 0
+; CHECK-NEXT: v_mov_b32_e32 v72, 0
+; CHECK-NEXT: v_mov_b32_e32 v79, 0
+; CHECK-NEXT: v_mov_b32_e32 v78, 0
+; CHECK-NEXT: v_mov_b32_e32 v77, 0
+; CHECK-NEXT: v_mov_b32_e32 v76, 0
+; CHECK-NEXT: v_mov_b32_e32 v83, 0
+; CHECK-NEXT: v_mov_b32_e32 v82, 0
+; CHECK-NEXT: v_mov_b32_e32 v81, 0
+; CHECK-NEXT: v_mov_b32_e32 v80, 0
+; CHECK-NEXT: v_mov_b32_e32 v87, 0
+; CHECK-NEXT: v_mov_b32_e32 v86, 0
+; CHECK-NEXT: v_mov_b32_e32 v85, 0
+; CHECK-NEXT: v_mov_b32_e32 v84, 0
+; CHECK-NEXT: v_mov_b32_e32 v91, 0
+; CHECK-NEXT: v_mov_b32_e32 v90, 0
+; CHECK-NEXT: v_mov_b32_e32 v89, 0
+; CHECK-NEXT: v_mov_b32_e32 v88, 0
+; CHECK-NEXT: v_mov_b32_e32 v95, 0
+; CHECK-NEXT: v_mov_b32_e32 v94, 0
+; CHECK-NEXT: v_mov_b32_e32 v93, 0
+; CHECK-NEXT: v_mov_b32_e32 v92, 0
; CHECK-NEXT: v_mov_b32_e32 v99, 0
; CHECK-NEXT: v_mov_b32_e32 v98, 0
; CHECK-NEXT: v_mov_b32_e32 v97, 0
; CHECK-NEXT: v_mov_b32_e32 v96, 0
-; CHECK-NEXT: v_mov_b32_e32 v159, 0
-; CHECK-NEXT: v_mov_b32_e32 v158, 0
-; CHECK-NEXT: v_mov_b32_e32 v157, 0
-; CHECK-NEXT: v_mov_b32_e32 v156, 0
-; CHECK-NEXT: v_mov_b32_e32 v155, 0
-; CHECK-NEXT: v_mov_b32_e32 v154, 0
-; CHECK-NEXT: v_mov_b32_e32 v153, 0
-; CHECK-NEXT: v_mov_b32_e32 v152, 0
-; CHECK-NEXT: v_mov_b32_e32 v151, 0
-; CHECK-NEXT: v_mov_b32_e32 v150, 0
-; CHECK-NEXT: v_mov_b32_e32 v149, 0
-; CHECK-NEXT: v_mov_b32_e32 v148, 0
-; CHECK-NEXT: v_mov_b32_e32 v147, 0
-; CHECK-NEXT: v_mov_b32_e32 v146, 0
-; CHECK-NEXT: v_mov_b32_e32 v145, 0
-; CHECK-NEXT: v_mov_b32_e32 v144, 0
-; CHECK-NEXT: v_mov_b32_e32 v143, 0
-; CHECK-NEXT: v_mov_b32_e32 v142, 0
-; CHECK-NEXT: v_mov_b32_e32 v141, 0
-; CHECK-NEXT: v_mov_b32_e32 v140, 0
-; CHECK-NEXT: v_mov_b32_e32 v139, 0
-; CHECK-NEXT: v_mov_b32_e32 v138, 0
-; CHECK-NEXT: v_mov_b32_e32 v137, 0
-; CHECK-NEXT: v_mov_b32_e32 v136, 0
-; CHECK-NEXT: v_mov_b32_e32 v135, 0
-; CHECK-NEXT: v_mov_b32_e32 v134, 0
-; CHECK-NEXT: v_mov_b32_e32 v133, 0
-; CHECK-NEXT: v_mov_b32_e32 v132, 0
+; CHECK-NEXT: v_mov_b32_e32 v103, 0
+; CHECK-NEXT: v_mov_b32_e32 v102, 0
+; CHECK-NEXT: v_mov_b32_e32 v101, 0
+; CHECK-NEXT: v_mov_b32_e32 v100, 0
+; CHECK-NEXT: v_mov_b32_e32 v107, 0
+; CHECK-NEXT: v_mov_b32_e32 v106, 0
+; CHECK-NEXT: v_mov_b32_e32 v105, 0
+; CHECK-NEXT: v_mov_b32_e32 v104, 0
+; CHECK-NEXT: v_mov_b32_e32 v111, 0
+; CHECK-NEXT: v_mov_b32_e32 v110, 0
+; CHECK-NEXT: v_mov_b32_e32 v109, 0
+; CHECK-NEXT: v_mov_b32_e32 v108, 0
+; CHECK-NEXT: v_mov_b32_e32 v115, 0
+; CHECK-NEXT: v_mov_b32_e32 v114, 0
+; CHECK-NEXT: v_mov_b32_e32 v113, 0
+; CHECK-NEXT: v_mov_b32_e32 v112, 0
+; CHECK-NEXT: v_mov_b32_e32 v119, 0
+; CHECK-NEXT: v_mov_b32_e32 v118, 0
+; CHECK-NEXT: v_mov_b32_e32 v117, 0
+; CHECK-NEXT: v_mov_b32_e32 v116, 0
+; CHECK-NEXT: v_mov_b32_e32 v123, 0
+; CHECK-NEXT: v_mov_b32_e32 v122, 0
+; CHECK-NEXT: v_mov_b32_e32 v121, 0
+; CHECK-NEXT: v_mov_b32_e32 v120, 0
+; CHECK-NEXT: v_mov_b32_e32 v127, 0
+; CHECK-NEXT: v_mov_b32_e32 v126, 0
+; CHECK-NEXT: v_mov_b32_e32 v125, 0
+; CHECK-NEXT: v_mov_b32_e32 v124, 0
; CHECK-NEXT: v_mov_b32_e32 v131, 0
; CHECK-NEXT: v_mov_b32_e32 v130, 0
; CHECK-NEXT: v_mov_b32_e32 v129, 0
; CHECK-NEXT: v_mov_b32_e32 v128, 0
-; CHECK-NEXT: v_mov_b32_e32 v191, 0
-; CHECK-NEXT: v_mov_b32_e32 v190, 0
-; CHECK-NEXT: v_mov_b32_e32 v189, 0
-; CHECK-NEXT: v_mov_b32_e32 v188, 0
-; CHECK-NEXT: v_mov_b32_e32 v187, 0
-; CHECK-NEXT: v_mov_b32_e32 v186, 0
-; CHECK-NEXT: v_mov_b32_e32 v185, 0
-; CHECK-NEXT: v_mov_b32_e32 v184, 0
-; CHECK-NEXT: v_mov_b32_e32 v183, 0
-; CHECK-NEXT: v_mov_b32_e32 v182, 0
-; CHECK-NEXT: v_mov_b32_e32 v181, 0
-; CHECK-NEXT: v_mov_b32_e32 v180, 0
-; CHECK-NEXT: v_mov_b32_e32 v179, 0
-; CHECK-NEXT: v_mov_b32_e32 v178, 0
-; CHECK-NEXT: v_mov_b32_e32 v177, 0
-; CHECK-NEXT: v_mov_b32_e32 v176, 0
-; CHECK-NEXT: v_mov_b32_e32 v175, 0
-; CHECK-NEXT: v_mov_b32_e32 v174, 0
-; CHECK-NEXT: v_mov_b32_e32 v173, 0
-; CHECK-NEXT: v_mov_b32_e32 v172, 0
-; CHECK-NEXT: v_mov_b32_e32 v171, 0
-; CHECK-NEXT: v_mov_b32_e32 v170, 0
-; CHECK-NEXT: v_mov_b32_e32 v169, 0
-; CHECK-NEXT: v_mov_b32_e32 v168, 0
-; CHECK-NEXT: v_mov_b32_e32 v167, 0
-; CHECK-NEXT: v_mov_b32_e32 v166, 0
-; CHECK-NEXT: v_mov_b32_e32 v165, 0
-; CHECK-NEXT: v_mov_b32_e32 v164, 0
+; CHECK-NEXT: v_mov_b32_e32 v135, 0
+; CHECK-NEXT: v_mov_b32_e32 v134, 0
+; CHECK-NEXT: v_mov_b32_e32 v133, 0
+; CHECK-NEXT: v_mov_b32_e32 v132, 0
+; CHECK-NEXT: v_mov_b32_e32 v139, 0
+; CHECK-NEXT: v_mov_b32_e32 v138, 0
+; CHECK-NEXT: v_mov_b32_e32 v137, 0
+; CHECK-NEXT: v_mov_b32_e32 v136, 0
+; CHECK-NEXT: v_mov_b32_e32 v143, 0
+; CHECK-NEXT: v_mov_b32_e32 v142, 0
+; CHECK-NEXT: v_mov_b32_e32 v141, 0
+; CHECK-NEXT: v_mov_b32_e32 v140, 0
+; CHECK-NEXT: v_mov_b32_e32 v147, 0
+; CHECK-NEXT: v_mov_b32_e32 v146, 0
+; CHECK-NEXT: v_mov_b32_e32 v145, 0
+; CHECK-NEXT: v_mov_b32_e32 v144, 0
+; CHECK-NEXT: v_mov_b32_e32 v151, 0
+; CHECK-NEXT: v_mov_b32_e32 v150, 0
+; CHECK-NEXT: v_mov_b32_e32 v149, 0
+; CHECK-NEXT: v_mov_b32_e32 v148, 0
+; CHECK-NEXT: v_mov_b32_e32 v155, 0
+; CHECK-NEXT: v_mov_b32_e32 v154, 0
+; CHECK-NEXT: v_mov_b32_e32 v153, 0
+; CHECK-NEXT: v_mov_b32_e32 v152, 0
+; CHECK-NEXT: v_mov_b32_e32 v159, 0
+; CHECK-NEXT: v_mov_b32_e32 v158, 0
+; CHECK-NEXT: v_mov_b32_e32 v157, 0
+; CHECK-NEXT: v_mov_b32_e32 v156, 0
; CHECK-NEXT: v_mov_b32_e32 v163, 0
; CHECK-NEXT: v_mov_b32_e32 v162, 0
; CHECK-NEXT: v_mov_b32_e32 v161, 0
; CHECK-NEXT: v_mov_b32_e32 v160, 0
-; CHECK-NEXT: v_mov_b32_e32 v223, 0
-; CHECK-NEXT: v_mov_b32_e32 v222, 0
-; CHECK-NEXT: v_mov_b32_e32 v221, 0
-; CHECK-NEXT: v_mov_b32_e32 v220, 0
-; CHECK-NEXT: v_mov_b32_e32 v219, 0
-; CHECK-NEXT: v_mov_b32_e32 v218, 0
-; CHECK-NEXT: v_mov_b32_e32 v217, 0
-; CHECK-NEXT: v_mov_b32_e32 v216, 0
-; CHECK-NEXT: v_mov_b32_e32 v215, 0
-; CHECK-NEXT: v_mov_b32_e32 v214, 0
-; CHECK-NEXT: v_mov_b32_e32 v213, 0
-; CHECK-NEXT: v_mov_b32_e32 v212, 0
-; CHECK-NEXT: v_mov_b32_e32 v211, 0
-; CHECK-NEXT: v_mov_b32_e32 v210, 0
-; CHECK-NEXT: v_mov_b32_e32 v209, 0
-; CHECK-NEXT: v_mov_b32_e32 v208, 0
-; CHECK-NEXT: v_mov_b32_e32 v207, 0
-; CHECK-NEXT: v_mov_b32_e32 v206, 0
-; CHECK-NEXT: v_mov_b32_e32 v205, 0
-; CHECK-NEXT: v_mov_b32_e32 v204, 0
-; CHECK-NEXT: v_mov_b32_e32 v203, 0
-; CHECK-NEXT: v_mov_b32_e32 v202, 0
-; CHECK-NEXT: v_mov_b32_e32 v201, 0
-; CHECK-NEXT: v_mov_b32_e32 v200, 0
-; CHECK-NEXT: v_mov_b32_e32 v199, 0
-; CHECK-NEXT: v_mov_b32_e32 v198, 0
-; CHECK-NEXT: v_mov_b32_e32 v197, 0
-; CHECK-NEXT: v_mov_b32_e32 v196, 0
+; CHECK-NEXT: v_mov_b32_e32 v167, 0
+; CHECK-NEXT: v_mov_b32_e32 v166, 0
+; CHECK-NEXT: v_mov_b32_e32 v165, 0
+; CHECK-NEXT: v_mov_b32_e32 v164, 0
+; CHECK-NEXT: v_mov_b32_e32 v171, 0
+; CHECK-NEXT: v_mov_b32_e32 v170, 0
+; CHECK-NEXT: v_mov_b32_e32 v169, 0
+; CHECK-NEXT: v_mov_b32_e32 v168, 0
+; CHECK-NEXT: v_mov_b32_e32 v175, 0
+; CHECK-NEXT: v_mov_b32_e32 v174, 0
+; CHECK-NEXT: v_mov_b32_e32 v173, 0
+; CHECK-NEXT: v_mov_b32_e32 v172, 0
+; CHECK-NEXT: v_mov_b32_e32 v179, 0
+; CHECK-NEXT: v_mov_b32_e32 v178, 0
+; CHECK-NEXT: v_mov_b32_e32 v177, 0
+; CHECK-NEXT: v_mov_b32_e32 v176, 0
+; CHECK-NEXT: v_mov_b32_e32 v183, 0
+; CHECK-NEXT: v_mov_b32_e32 v182, 0
+; CHECK-NEXT: v_mov_b32_e32 v181, 0
+; CHECK-NEXT: v_mov_b32_e32 v180, 0
+; CHECK-NEXT: v_mov_b32_e32 v187, 0
+; CHECK-NEXT: v_mov_b32_e32 v186, 0
+; CHECK-NEXT: v_mov_b32_e32 v185, 0
+; CHECK-NEXT: v_mov_b32_e32 v184, 0
+; CHECK-NEXT: v_mov_b32_e32 v191, 0
+; CHECK-NEXT: v_mov_b32_e32 v190, 0
+; CHECK-NEXT: v_mov_b32_e32 v189, 0
+; CHECK-NEXT: v_mov_b32_e32 v188, 0
; CHECK-NEXT: v_mov_b32_e32 v195, 0
; CHECK-NEXT: v_mov_b32_e32 v194, 0
; CHECK-NEXT: v_mov_b32_e32 v193, 0
; CHECK-NEXT: v_mov_b32_e32 v192, 0
+; CHECK-NEXT: v_mov_b32_e32 v199, 0
+; CHECK-NEXT: v_mov_b32_e32 v198, 0
+; CHECK-NEXT: v_mov_b32_e32 v197, 0
+; CHECK-NEXT: v_mov_b32_e32 v196, 0
+; CHECK-NEXT: v_mov_b32_e32 v203, 0
+; CHECK-NEXT: v_mov_b32_e32 v202, 0
+; CHECK-NEXT: v_mov_b32_e32 v201, 0
+; CHECK-NEXT: v_mov_b32_e32 v200, 0
+; CHECK-NEXT: v_mov_b32_e32 v207, 0
+; CHECK-NEXT: v_mov_b32_e32 v206, 0
+; CHECK-NEXT: v_mov_b32_e32 v205, 0
+; CHECK-NEXT: v_mov_b32_e32 v204, 0
+; CHECK-NEXT: v_mov_b32_e32 v211, 0
+; CHECK-NEXT: v_mov_b32_e32 v210, 0
+; CHECK-NEXT: v_mov_b32_e32 v209, 0
+; CHECK-NEXT: v_mov_b32_e32 v208, 0
+; CHECK-NEXT: v_mov_b32_e32 v215, 0
+; CHECK-NEXT: v_mov_b32_e32 v214, 0
+; CHECK-NEXT: v_mov_b32_e32 v213, 0
+; CHECK-NEXT: v_mov_b32_e32 v212, 0
+; CHECK-NEXT: v_mov_b32_e32 v219, 0
+; CHECK-NEXT: v_mov_b32_e32 v218, 0
+; CHECK-NEXT: v_mov_b32_e32 v217, 0
+; CHECK-NEXT: v_mov_b32_e32 v216, 0
+; CHECK-NEXT: v_mov_b32_e32 v223, 0
+; CHECK-NEXT: v_mov_b32_e32 v222, 0
+; CHECK-NEXT: v_mov_b32_e32 v221, 0
+; CHECK-NEXT: v_mov_b32_e32 v220, 0
; CHECK-NEXT: v_mov_b32_e32 v243, 0
; CHECK-NEXT: v_mov_b32_e32 v242, 0
; CHECK-NEXT: v_mov_b32_e32 v241, 0
@@ -258,22 +258,14 @@ define amdgpu_kernel void @single_exit_copy(
; CHECK-NEXT: v_mov_b32_e32 v246, 0
; CHECK-NEXT: v_mov_b32_e32 v245, 0
; CHECK-NEXT: v_mov_b32_e32 v244, 0
-; CHECK-NEXT: v_mov_b32_e32 v251, 0
-; CHECK-NEXT: v_mov_b32_e32 v250, 0
-; CHECK-NEXT: v_mov_b32_e32 v249, 0
-; CHECK-NEXT: v_mov_b32_e32 v248, 0
; CHECK-NEXT: v_mov_b32_e32 v255, 0
; CHECK-NEXT: v_mov_b32_e32 v254, 0
; CHECK-NEXT: v_mov_b32_e32 v253, 0
; CHECK-NEXT: v_mov_b32_e32 v252, 0
-; CHECK-NEXT: v_mov_b32_e32 v239, 0
-; CHECK-NEXT: v_mov_b32_e32 v238, 0
-; CHECK-NEXT: v_mov_b32_e32 v237, 0
-; CHECK-NEXT: v_mov_b32_e32 v236, 0
-; CHECK-NEXT: v_mov_b32_e32 v235, 0
-; CHECK-NEXT: v_mov_b32_e32 v234, 0
-; CHECK-NEXT: v_mov_b32_e32 v233, 0
-; CHECK-NEXT: v_mov_b32_e32 v232, 0
+; CHECK-NEXT: v_mov_b32_e32 v251, 0
+; CHECK-NEXT: v_mov_b32_e32 v250, 0
+; CHECK-NEXT: v_mov_b32_e32 v249, 0
+; CHECK-NEXT: v_mov_b32_e32 v248, 0
; CHECK-NEXT: v_mov_b32_e32 v231, 0
; CHECK-NEXT: v_mov_b32_e32 v230, 0
; CHECK-NEXT: v_mov_b32_e32 v229, 0
@@ -282,6 +274,14 @@ define amdgpu_kernel void @single_exit_copy(
; CHECK-NEXT: v_mov_b32_e32 v226, 0
; CHECK-NEXT: v_mov_b32_e32 v225, 0
; CHECK-NEXT: v_mov_b32_e32 v224, 0
+; CHECK-NEXT: v_mov_b32_e32 v235, 0
+; CHECK-NEXT: v_mov_b32_e32 v234, 0
+; CHECK-NEXT: v_mov_b32_e32 v233, 0
+; CHECK-NEXT: v_mov_b32_e32 v232, 0
+; CHECK-NEXT: v_mov_b32_e32 v239, 0
+; CHECK-NEXT: v_mov_b32_e32 v238, 0
+; CHECK-NEXT: v_mov_b32_e32 v237, 0
+; CHECK-NEXT: v_mov_b32_e32 v236, 0
; CHECK-NEXT: v_accvgpr_write_b32 a0, 0
; CHECK-NEXT: v_accvgpr_write_b32 a1, 0
; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
@@ -310,133 +310,133 @@ define amdgpu_kernel void @single_exit_copy(
; CHECK-NEXT: v_add_f32_e64 v246, v246, v246
; CHECK-NEXT: v_add_f32_e64 v247, v247, v247
; CHECK-NEXT: v_pk_add_f32 v[244:245], v[244:245], v[244:245]
-; CHECK-NEXT: v_pk_add_f32 v[250:251], v[250:251], v[250:251]
-; CHECK-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], a[4:7], a[8:11], a[0:3]
-; CHECK-NEXT: v_add_f32_e64 v248, v248, v248
-; CHECK-NEXT: v_add_f32_e64 v249, v249, v249
; CHECK-NEXT: v_pk_add_f32 v[254:255], v[254:255], v[254:255]
-; CHECK-NEXT: v_pk_add_f32 v[252:253], v[252:253], v[252:253]
-; CHECK-NEXT: v_pk_add_f32 v[238:239], v[238:239], v[238:239]
-; CHECK-NEXT: v_pk_add_f32 v[236:237], v[236:237], v[236:237]
-; CHECK-NEXT: v_pk_add_f32 v[234:235], v[234:235], v[234:235]
-; CHECK-NEXT: v_pk_add_f32 v[232:233], v[232:233], v[232:233]
+; CHECK-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], a[4:7], a[8:11], a[0:3]
+; CHECK-NEXT: v_add_f32_e64 v252, v252, v252
+; CHECK-NEXT: v_add_f32_e64 v253, v253, v253
+; CHECK-NEXT: v_pk_add_f32 v[250:251], v[250:251], v[250:251]
+; CHECK-NEXT: v_pk_add_f32 v[248:249], v[248:249], v[248:249]
; CHECK-NEXT: v_pk_add_f32 v[230:231], v[230:231], v[230:231]
; CHECK-NEXT: v_pk_add_f32 v[228:229], v[228:229], v[228:229]
; CHECK-NEXT: v_pk_add_f32 v[226:227], v[226:227], v[226:227]
; CHECK-NEXT: v_pk_add_f32 v[224:225], v[224:225], v[224:225]
-; CHECK-NEXT: v_pk_add_f32 v[222:223], v[222:223], v[222:223]
-; CHECK-NEXT: v_pk_add_f32 v[220:221], v[220:221], v[220:221]
-; CHECK-NEXT: v_pk_add_f32 v[218:219], v[218:219], v[218:219]
-; CHECK-NEXT: v_pk_add_f32 v[216:217], v[216:217], v[216:217]
-; CHECK-NEXT: v_pk_add_f32 v[214:215], v[214:215], v[214:215]
-; CHECK-NEXT: v_pk_add_f32 v[212:213], v[212:213], v[212:213]
-; CHECK-NEXT: v_pk_add_f32 v[210:211], v[210:211], v[210:211]
-; CHECK-NEXT: v_pk_add_f32 v[208:209], v[208:209], v[208:209]
-; CHECK-NEXT: v_pk_add_f32 v[206:207], v[206:207], v[206:207]
-; CHECK-NEXT: v_pk_add_f32 v[204:205], v[204:205], v[204:205]
-; CHECK-NEXT: v_pk_add_f32 v[202:203], v[202:203], v[202:203]
-; CHECK-NEXT: v_pk_add_f32 v[200:201], v[200:201], v[200:201]
-; CHECK-NEXT: v_pk_add_f32 v[198:199], v[198:199], v[198:199]
-; CHECK-NEXT: v_pk_add_f32 v[196:197], v[196:197], v[196:197]
+; CHECK-NEXT: v_pk_add_f32 v[234:235], v[234:235], v[234:235]
+; CHECK-NEXT: v_pk_add_f32 v[232:233], v[232:233], v[232:233]
+; CHECK-NEXT: v_pk_add_f32 v[238:239], v[238:239], v[238:239]
+; CHECK-NEXT: v_pk_add_f32 v[236:237], v[236:237], v[236:237]
; CHECK-NEXT: v_pk_add_f32 v[194:195], v[194:195], v[194:195]
; CHECK-NEXT: v_pk_add_f32 v[192:193], v[192:193], v[192:193]
-; CHECK-NEXT: v_pk_add_f32 v[190:191], v[190:191], v[190:191]
-; CHECK-NEXT: v_pk_add_f32 v[188:189], v[188:189], v[188:189]
-; CHECK-NEXT: v_pk_add_f32 v[186:187], v[186:187], v[186:187]
-; CHECK-NEXT: v_pk_add_f32 v[184:185], v[184:185], v[184:185]
-; CHECK-NEXT: v_pk_add_f32 v[182:183], v[182:183], v[182:183]
-; CHECK-NEXT: v_pk_add_f32 v[180:181], v[180:181], v[180:181]
-; CHECK-NEXT: v_pk_add_f32 v[178:179], v[178:179], v[178:179]
-; CHECK-NEXT: v_pk_add_f32 v[176:177], v[176:177], v[176:177]
-; CHECK-NEXT: v_pk_add_f32 v[174:175], v[174:175], v[174:175]
-; CHECK-NEXT: v_pk_add_f32 v[172:173], v[172:173], v[172:173]
-; CHECK-NEXT: v_pk_add_f32 v[170:171], v[170:171], v[170:171]
-; CHECK-NEXT: v_pk_add_f32 v[168:169], v[168:169], v[168:169]
-; CHECK-NEXT: v_pk_add_f32 v[166:167], v[166:167], v[166:167]
-; CHECK-NEXT: v_pk_add_f32 v[164:165], v[164:165], v[164:165]
+; CHECK-NEXT: v_pk_add_f32 v[198:199], v[198:199], v[198:199]
+; CHECK-NEXT: v_pk_add_f32 v[196:197], v[196:197], v[196:197]
+; CHECK-NEXT: v_pk_add_f32 v[202:203], v[202:203], v[202:203]
+; CHECK-NEXT: v_pk_add_f32 v[200:201], v[200:201], v[200:201]
+; CHECK-NEXT: v_pk_add_f32 v[206:207], v[206:207], v[206:207]
+; CHECK-NEXT: v_pk_add_f32 v[204:205], v[204:205], v[204:205]
+; CHECK-NEXT: v_pk_add_f32 v[210:211], v[210:211], v[210:211]
+; CHECK-NEXT: v_pk_add_f32 v[208:209], v[208:209], v[208:209]
+; CHECK-NEXT: v_pk_add_f32 v[214:215], v[214:215], v[214:215]
+; CHECK-NEXT: v_pk_add_f32 v[212:213], v[212:213], v[212:213]
+; CHECK-NEXT: v_pk_add_f32 v[218:219], v[218:219], v[218:219]
+; CHECK-NEXT: v_pk_add_f32 v[216:217], v[216:217], v[216:217]
+; CHECK-NEXT: v_pk_add_f32 v[222:223], v[222:223], v[222:223]
+; CHECK-NEXT: v_pk_add_f32 v[220:221], v[220:221], v[220:221]
; CHECK-NEXT: v_pk_add_f32 v[162:163], v[162:163], v[162:163]
; CHECK-NEXT: v_pk_add_f32 v[160:161], v[160:161], v[160:161]
-; CHECK-NEXT: v_pk_add_f32 v[158:159], v[158:159], v[158:159]
-; CHECK-NEXT: v_pk_add_f32 v[156:157], v[156:157], v[156:157]
-; CHECK-NEXT: v_pk_add_f32 v[154:155], v[154:155], v[154:155]
-; CHECK-NEXT: v_pk_add_f32 v[152:153], v[152:153], v[152:153]
-; CHECK-NEXT: v_pk_add_f32 v[150:151], v[150:151], v[150:151]
-; CHECK-NEXT: v_pk_add_f32 v[148:149], v[148:149], v[148:149]
-; CHECK-NEXT: v_pk_add_f32 v[146:147], v[146:147], v[146:147]
-; CHECK-NEXT: v_pk_add_f32 v[144:145], v[144:145], v[144:145]
-; CHECK-NEXT: v_pk_add_f32 v[142:143], v[142:143], v[142:143]
-; CHECK-NEXT: v_pk_add_f32 v[140:141], v[140:141], v[140:141]
-; CHECK-NEXT: v_pk_add_f32 v[138:139], v[138:139], v[138:139]
-; CHECK-NEXT: v_pk_add_f32 v[136:137], v[136:137], v[136:137]
-; CHECK-NEXT: v_pk_add_f32 v[134:135], v[134:135], v[134:135]
-; CHECK-NEXT: v_pk_add_f32 v[132:133], v[132:133], v[132:133]
+; CHECK-NEXT: v_pk_add_f32 v[166:167], v[166:167], v[166:167]
+; CHECK-NEXT: v_pk_add_f32 v[164:165], v[164:165], v[164:165]
+; CHECK-NEXT: v_pk_add_f32 v[170:171], v[170:171], v[170:171]
+; CHECK-NEXT: v_pk_add_f32 v[168:169], v[168:169], v[168:169]
+; CHECK-NEXT: v_pk_add_f32 v[174:175], v[174:175], v[174:175]
+; CHECK-NEXT: v_pk_add_f32 v[172:173], v[172:173], v[172:173]
+; CHECK-NEXT: v_pk_add_f32 v[178:179], v[178:179], v[178:179]
+; CHECK-NEXT: v_pk_add_f32 v[176:177], v[176:177], v[176:177]
+; CHECK-NEXT: v_pk_add_f32 v[182:183], v[182:183], v[182:183]
+; CHECK-NEXT: v_pk_add_f32 v[180:181], v[180:181], v[180:181]
+; CHECK-NEXT: v_pk_add_f32 v[186:187], v[186:187], v[186:187]
+; CHECK-NEXT: v_pk_add_f32 v[184:185], v[184:185], v[184:185]
+; CHECK-NEXT: v_pk_add_f32 v[190:191], v[190:191], v[190:191]
+; CHECK-NEXT: v_pk_add_f32 v[188:189], v[188:189], v[188:189]
; CHECK-NEXT: v_pk_add_f32 v[130:131], v[130:131], v[130:131]
; CHECK-NEXT: v_pk_add_f32 v[128:129], v[128:129], v[128:129]
-; CHECK-NEXT: v_pk_add_f32 v[126:127], v[126:127], v[126:127]
-; CHECK-NEXT: v_pk_add_f32 v[124:125], v[124:125], v[124:125]
-; CHECK-NEXT: v_pk_add_f32 v[122:123], v[122:123], v[122:123]
-; CHECK-NEXT: v_pk_add_f32 v[120:121], v[120:121], v[120:121]
-; CHECK-NEXT: v_pk_add_f32 v[118:119], v[118:119], v[118:119]
-; CHECK-NEXT: v_pk_add_f32 v[116:117], v[116:117], v[116:117]
-; CHECK-NEXT: v_pk_add_f32 v[114:115], v[114:115], v[114:115]
-; CHECK-NEXT: v_pk_add_f32 v[112:113], v[112:113], v[112:113]
-; CHECK-NEXT: v_pk_add_f32 v[110:111], v[110:111], v[110:111]
-; CHECK-NEXT: v_pk_add_f32 v[108:109], v[108:109], v[108:109]
-; CHECK-NEXT: v_pk_add_f32 v[106:107], v[106:107], v[106:107]
-; CHECK-NEXT: v_pk_add_f32 v[104:105], v[104:105], v[104:105]
-; CHECK-NEXT: v_pk_add_f32 v[102:103], v[102:103], v[102:103]
-; CHECK-NEXT: v_pk_add_f32 v[100:101], v[100:101], v[100:101]
+; CHECK-NEXT: v_pk_add_f32 v[134:135], v[134:135], v[134:135]
+; CHECK-NEXT: v_pk_add_f32 v[132:133], v[132:133], v[132:133]
+; CHECK-NEXT: v_pk_add_f32 v[138:139], v[138:139], v[138:139]
+; CHECK-NEXT: v_pk_add_f32 v[136:137], v[136:137], v[136:137]
+; CHECK-NEXT: v_pk_add_f32 v[142:143], v[142:143], v[142:143]
+; CHECK-NEXT: v_pk_add_f32 v[140:141], v[140:141], v[140:141]
+; CHECK-NEXT: v_pk_add_f32 v[146:147], v[146:147], v[146:147]
+; CHECK-NEXT: v_pk_add_f32 v[144:145], v[144:145], v[144:145]
+; CHECK-NEXT: v_pk_add_f32 v[150:151], v[150:151], v[150:151]
+; CHECK-NEXT: v_pk_add_f32 v[148:149], v[148:149], v[148:149]
+; CHECK-NEXT: v_pk_add_f32 v[154:155], v[154:155], v[154:155]
+; CHECK-NEXT: v_pk_add_f32 v[152:153], v[152:153], v[152:153]
+; CHECK-NEXT: v_pk_add_f32 v[158:159], v[158:159], v[158:159]
+; CHECK-NEXT: v_pk_add_f32 v[156:157], v[156:157], v[156:157]
; CHECK-NEXT: v_pk_add_f32 v[98:99], v[98:99], v[98:99]
; CHECK-NEXT: v_pk_add_f32 v[96:97], v[96:97], v[96:97]
-; CHECK-NEXT: v_pk_add_f32 v[94:95], v[94:95], v[94:95]
-; CHECK-NEXT: v_pk_add_f32 v[92:93], v[92:93], v[92:93]
-; CHECK-NEXT: v_pk_add_f32 v[90:91], v[90:91], v[90:91]
-; CHECK-NEXT: v_pk_add_f32 v[88:89], v[88:89], v[88:89]
-; CHECK-NEXT: v_pk_add_f32 v[86:87], v[86:87], v[86:87]
-; CHECK-NEXT: v_pk_add_f32 v[84:85], v[84:85], v[84:85]
-; CHECK-NEXT: v_pk_add_f32 v[82:83], v[82:83], v[82:83]
-; CHECK-NEXT: v_pk_add_f32 v[80:81], v[80:81], v[80:81]
-; CHECK-NEXT: v_pk_add_f32 v[78:79], v[78:79], v[78:79]
-; CHECK-NEXT: v_pk_add_f32 v[76:77], v[76:77], v[76:77]
-; CHECK-NEXT: v_pk_add_f32 v[74:75], v[74:75], v[74:75]
-; CHECK-NEXT: v_pk_add_f32 v[72:73], v[72:73], v[72:73]
-; CHECK-NEXT: v_pk_add_f32 v[70:71], v[70:71], v[70:71]
-; CHECK-NEXT: v_pk_add_f32 v[68:69], v[68:69], v[68:69]
+; CHECK-NEXT: v_pk_add_f32 v[102:103], v[102:103], v[102:103]
+; CHECK-NEXT: v_pk_add_f32 v[100:101], v[100:101], v[100:101]
+; CHECK-NEXT: v_pk_add_f32 v[106:107], v[106:107], v[106:107]
+; CHECK-NEXT: v_pk_add_f32 v[104:105], v[104:105], v[104:105]
+; CHECK-NEXT: v_pk_add_f32 v[110:111], v[110:111], v[110:111]
+; CHECK-NEXT: v_pk_add_f32 v[108:109], v[108:109], v[108:109]
+; CHECK-NEXT: v_pk_add_f32 v[114:115], v[114:115], v[114:115]
+; CHECK-NEXT: v_pk_add_f32 v[112:113], v[112:113], v[112:113]
+; CHECK-NEXT: v_pk_add_f32 v[118:119], v[118:119], v[118:119]
+; CHECK-NEXT: v_pk_add_f32 v[116:117], v[116:117], v[116:117]
+; CHECK-NEXT: v_pk_add_f32 v[122:123], v[122:123], v[122:123]
+; CHECK-NEXT: v_pk_add_f32 v[120:121], v[120:121], v[120:121]
+; CHECK-NEXT: v_pk_add_f32 v[126:127], v[126:127], v[126:127]
+; CHECK-NEXT: v_pk_add_f32 v[124:125], v[124:125], v[124:125]
; CHECK-NEXT: v_pk_add_f32 v[66:67], v[66:67], v[66:67]
; CHECK-NEXT: v_pk_add_f32 v[64:65], v[64:65], v[64:65]
-; CHECK-NEXT: v_pk_add_f32 v[62:63], v[62:63], v[62:63]
-; CHECK-NEXT: v_pk_add_f32 v[60:61], v[60:61], v[60:61]
-; CHECK-NEXT: v_pk_add_f32 v[58:59], v[58:59], v[58:59]
-; CHECK-NEXT: v_pk_add_f32 v[56:57], v[56:57], v[56:57]
-; CHECK-NEXT: v_pk_add_f32 v[54:55], v[54:55], v[54:55]
-; CHECK-NEXT: v_pk_add_f32 v[52:53], v[52:53], v[52:53]
-; CHECK-NEXT: v_pk_add_f32 v[50:51], v[50:51], v[50:51]
-; CHECK-NEXT: v_pk_add_f32 v[48:49], v[48:49], v[48:49]
-; CHECK-NEXT: v_pk_add_f32 v[46:47], v[46:47], v[46:47]
-; CHECK-NEXT: v_pk_add_f32 v[44:45], v[44:45], v[44:45]
-; CHECK-NEXT: v_pk_add_f32 v[42:43], v[42:43], v[42:43]
-; CHECK-NEXT: v_pk_add_f32 v[40:41], v[40:41], v[40:41]
-; CHECK-NEXT: v_pk_add_f32 v[38:39], v[38:39], v[38:39]
-; CHECK-NEXT: v_pk_add_f32 v[36:37], v[36:37], v[36:37]
+; CHECK-NEXT: v_pk_add_f32 v[70:71], v[70:71], v[70:71]
+; CHECK-NEXT: v_pk_add_f32 v[68:69], v[68:69], v[68:69]
+; CHECK-NEXT: v_pk_add_f32 v[74:75], v[74:75], v[74:75]
+; CHECK-NEXT: v_pk_add_f32 v[72:73], v[72:73], v[72:73]
+; CHECK-NEXT: v_pk_add_f32 v[78:79], v[78:79], v[78:79]
+; CHECK-NEXT: v_pk_add_f32 v[76:77], v[76:77], v[76:77]
+; CHECK-NEXT: v_pk_add_f32 v[82:83], v[82:83], v[82:83]
+; CHECK-NEXT: v_pk_add_f32 v[80:81], v[80:81], v[80:81]
+; CHECK-NEXT: v_pk_add_f32 v[86:87], v[86:87], v[86:87]
+; CHECK-NEXT: v_pk_add_f32 v[84:85], v[84:85], v[84:85]
+; CHECK-NEXT: v_pk_add_f32 v[90:91], v[90:91], v[90:91]
+; CHECK-NEXT: v_pk_add_f32 v[88:89], v[88:89], v[88:89]
+; CHECK-NEXT: v_pk_add_f32 v[94:95], v[94:95], v[94:95]
+; CHECK-NEXT: v_pk_add_f32 v[92:93], v[92:93], v[92:93]
; CHECK-NEXT: v_pk_add_f32 v[34:35], v[34:35], v[34:35]
; CHECK-NEXT: v_pk_add_f32 v[32:33], v[32:33], v[32:33]
-; CHECK-NEXT: v_pk_add_f32 v[30:31], v[30:31], v[30:31]
-; CHECK-NEXT: v_pk_add_f32 v[28:29], v[28:29], v[28:29]
-; CHECK-NEXT: v_pk_add_f32 v[26:27], v[26:27], v[26:27]
-; CHECK-NEXT: v_pk_add_f32 v[24:25], v[24:25], v[24:25]
-; CHECK-NEXT: v_pk_add_f32 v[22:23], v[22:23], v[22:23]
-; CHECK-NEXT: v_pk_add_f32 v[20:21], v[20:21], v[20:21]
-; CHECK-NEXT: v_pk_add_f32 v[18:19], v[18:19], v[18:19]
-; CHECK-NEXT: v_pk_add_f32 v[16:17], v[16:17], v[16:17]
-; CHECK-NEXT: v_pk_add_f32 v[14:15], v[14:15], v[14:15]
-; CHECK-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[12:13]
-; CHECK-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[10:11]
-; CHECK-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[8:9]
+; CHECK-NEXT: v_pk_add_f32 v[38:39], v[38:39], v[38:39]
+; CHECK-NEXT: v_pk_add_f32 v[36:37], v[36:37], v[36:37]
+; CHECK-NEXT: v_pk_add_f32 v[42:43], v[42:43], v[42:43]
+; CHECK-NEXT: v_pk_add_f32 v[40:41], v[40:41], v[40:41]
+; CHECK-NEXT: v_pk_add_f32 v[46:47], v[46:47], v[46:47]
+; CHECK-NEXT: v_pk_add_f32 v[44:45], v[44:45], v[44:45]
+; CHECK-NEXT: v_pk_add_f32 v[50:51], v[50:51], v[50:51]
+; CHECK-NEXT: v_pk_add_f32 v[48:49], v[48:49], v[48:49]
+; CHECK-NEXT: v_pk_add_f32 v[54:55], v[54:55], v[54:55]
+; CHECK-NEXT: v_pk_add_f32 v[52:53], v[52:53], v[52:53]
+; CHECK-NEXT: v_pk_add_f32 v[58:59], v[58:59], v[58:59]
+; CHECK-NEXT: v_pk_add_f32 v[56:57], v[56:57], v[56:57]
+; CHECK-NEXT: v_pk_add_f32 v[62:63], v[62:63], v[62:63]
+; CHECK-NEXT: v_pk_add_f32 v[60:61], v[60:61], v[60:61]
+; CHECK-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[0:1]
; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[6:7]
; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[4:5]
-; CHECK-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[10:11]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[8:9]
+; CHECK-NEXT: v_pk_add_f32 v[14:15], v[14:15], v[14:15]
+; CHECK-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[12:13]
+; CHECK-NEXT: v_pk_add_f32 v[18:19], v[18:19], v[18:19]
+; CHECK-NEXT: v_pk_add_f32 v[16:17], v[16:17], v[16:17]
+; CHECK-NEXT: v_pk_add_f32 v[22:23], v[22:23], v[22:23]
+; CHECK-NEXT: v_pk_add_f32 v[20:21], v[20:21], v[20:21]
+; CHECK-NEXT: v_pk_add_f32 v[26:27], v[26:27], v[26:27]
+; CHECK-NEXT: v_pk_add_f32 v[24:25], v[24:25], v[24:25]
+; CHECK-NEXT: v_pk_add_f32 v[30:31], v[30:31], v[30:31]
; CHECK-NEXT: s_cmp_lt_i32 s1, s0
-; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[0:1]
+; CHECK-NEXT: v_pk_add_f32 v[28:29], v[28:29], v[28:29]
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: .Ltmp1:
@@ -458,87 +458,82 @@ define amdgpu_kernel void @single_exit_copy(
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v0, v[240:243], s[0:1] offset:2032
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: global_store_dwordx4 v0, v[244:247], s[0:1] offset:2016
-; CHECK-NEXT: global_store_dwordx4 v0, v[248:251], s[0:1] offset:2000
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: global_store_dwordx4 v0, v[252:255], s[0:1] offset:2000
+; CHECK-NEXT: global_store_dwordx4 v0, v[248:251], s[0:1] offset:1984
+; CHECK-NEXT: global_store_dwordx4 v0, v[228:231], s[0:1] offset:1968
; CHECK-NEXT: v_accvgpr_read_b32 v243, a3
; CHECK-NEXT: v_accvgpr_read_b32 v242, a2
; CHECK-NEXT: v_accvgpr_read_b32 v241, a1
; CHECK-NEXT: v_accvgpr_read_b32 v240, a0
-; CHECK-NEXT: global_store_dwordx4 v0, v[252:255], s[0:1] offset:1984
; CHECK-NEXT: .loc 1 20 1 ; test:20:1
-; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_mfma_f32_16x16x32_f16 v[244:247], a[4:7], a[8:11], v[240:243]
-; CHECK-NEXT: s_nop 7
-; CHECK-NEXT: v_mov_b32_e32 v245, 0
-; CHECK-NEXT: global_store_dwordx4 v245, v[236:239], s[0:1] offset:1968
; CHECK-NEXT: .loc 1 30 1 ; test:30:1
-; CHECK-NEXT: s_nop 1
-; CHECK-NEXT: v_mfma_f32_16x16x32_f16 v[236:239], a[8:11], a[4:7], v[240:243]
+; CHECK-NEXT: v_mfma_f32_16x16x32_f16 v[228:231], a[8:11], a[4:7], v[240:243]
; CHECK-NEXT: s_nop 7
-; CHECK-NEXT: v_add_f32_e32 v238, 1.0, v244
-; CHECK-NEXT: v_add_f32_e32 v239, 2.0, v236
-; CHECK-NEXT: global_store_dwordx2 v245, v[238:239], s[0:1]
-; CHECK-NEXT: global_store_dwordx4 v245, v[232:235], s[0:1] offset:1952
-; CHECK-NEXT: global_store_dwordx4 v245, v[228:231], s[0:1] offset:1936
-; CHECK-NEXT: global_store_dwordx4 v245, v[224:227], s[0:1] offset:1920
-; CHECK-NEXT: global_store_dwordx4 v245, v[220:223], s[0:1] offset:368
-; CHECK-NEXT: global_store_dwordx4 v245, v[216:219], s[0:1] offset:352
-; CHECK-NEXT: global_store_dwordx4 v245, v[212:215], s[0:1] offset:336
-; CHECK-NEXT: global_store_dwordx4 v245, v[208:211], s[0:1] offset:320
-; CHECK-NEXT: global_store_dwordx4 v245, v[204:207], s[0:1] offset:304
-; CHECK-NEXT: global_store_dwordx4 v245, v[200:203], s[0:1] offset:288
-; CHECK-NEXT: global_store_dwordx4 v245, v[196:199], s[0:1] offset:272
-; CHECK-NEXT: global_store_dwordx4 v245, v[192:195], s[0:1] offset:256
-; CHECK-NEXT: global_store_dwordx4 v245, v[188:191], s[0:1] offset:496
-; CHECK-NEXT: global_store_dwordx4 v245, v[184:187], s[0:1] offset:480
-; CHECK-NEXT: global_store_dwordx4 v245, v[180:183], s[0:1] offset:464
-; CHECK-NEXT: global_store_dwordx4 v245, v[176:179], s[0:1] offset:448
-; CHECK-NEXT: global_store_dwordx4 v245, v[172:175], s[0:1] offset:432
-; CHECK-NEXT: global_store_dwordx4 v245, v[168:171], s[0:1] offset:416
-; CHECK-NEXT: global_store_dwordx4 v245, v[164:167], s[0:1] offset:400
-; CHECK-NEXT: global_store_dwordx4 v245, v[160:163], s[0:1] offset:384
-; CHECK-NEXT: global_store_dwordx4 v245, v[156:159], s[0:1] offset:624
-; CHECK-NEXT: global_store_dwordx4 v245, v[152:155], s[0:1] offset:608
-; CHECK-NEXT: global_store_dwordx4 v245, v[148:151], s[0:1] offset:592
-; CHECK-NEXT: global_store_dwordx4 v245, v[144:147], s[0:1] offset:576
-; CHECK-NEXT: global_store_dwordx4 v245, v[140:143], s[0:1] offset:560
-; CHECK-NEXT: global_store_dwordx4 v245, v[136:139], s[0:1] offset:544
-; CHECK-NEXT: global_store_dwordx4 v245, v[132:135], s[0:1] offset:528
-; CHECK-NEXT: global_store_dwordx4 v245, v[128:131], s[0:1] offset:512
-; CHECK-NEXT: global_store_dwordx4 v245, v[124:127], s[0:1] offset:752
-; CHECK-NEXT: global_store_dwordx4 v245, v[120:123], s[0:1] offset:736
-; CHECK-NEXT: global_store_dwordx4 v245, v[116:119], s[0:1] offset:720
-; CHECK-NEXT: global_store_dwordx4 v245, v[112:115], s[0:1] offset:704
-; CHECK-NEXT: global_store_dwordx4 v245, v[108:111], s[0:1] offset:688
-; CHECK-NEXT: global_store_dwordx4 v245, v[104:107], s[0:1] offset:672
-; CHECK-NEXT: global_store_dwordx4 v245, v[100:103], s[0:1] offset:656
-; CHECK-NEXT: global_store_dwordx4 v245, v[96:99], s[0:1] offset:640
-; CHECK-NEXT: global_store_dwordx4 v245, v[92:95], s[0:1] offset:880
-; CHECK-NEXT: global_store_dwordx4 v245, v[88:91], s[0:1] offset:864
-; CHECK-NEXT: global_store_dwordx4 v245, v[84:87], s[0:1] offset:848
-; CHECK-NEXT: global_store_dwordx4 v245, v[80:83], s[0:1] offset:832
-; CHECK-NEXT: global_store_dwordx4 v245, v[76:79], s[0:1] offset:816
-; CHECK-NEXT: global_store_dwordx4 v245, v[72:75], s[0:1] offset:800
-; CHECK-NEXT: global_store_dwordx4 v245, v[68:71], s[0:1] offset:784
-; CHECK-NEXT: global_store_dwordx4 v245, v[64:67], s[0:1] offset:768
-; CHECK-NEXT: global_store_dwordx4 v245, v[60:63], s[0:1] offset:1008
-; CHECK-NEXT: global_store_dwordx4 v245, v[56:59], s[0:1] offset:992
-; CHECK-NEXT: global_store_dwordx4 v245, v[52:55], s[0:1] offset:976
-; CHECK-NEXT: global_store_dwordx4 v245, v[48:51], s[0:1] offset:960
-; CHECK-NEXT: global_store_dwordx4 v245, v[44:47], s[0:1] offset:944
-; CHECK-NEXT: global_store_dwordx4 v245, v[40:43], s[0:1] offset:928
-; CHECK-NEXT: global_store_dwordx4 v245, v[36:39], s[0:1] offset:912
-; CHECK-NEXT: global_store_dwordx4 v245, v[32:35], s[0:1] offset:896
-; CHECK-NEXT: global_store_dwordx4 v245, v[28:31], s[0:1] offset:1136
-; CHECK-NEXT: global_store_dwordx4 v245, v[24:27], s[0:1] offset:1120
-; CHECK-NEXT: global_store_dwordx4 v245, v[20:23], s[0:1] offset:1104
-; CHECK-NEXT: global_store_dwordx4 v245, v[16:19], s[0:1] offset:1088
-; CHECK-NEXT: global_store_dwordx4 v245, v[12:15], s[0:1] offset:1072
-; CHECK-NEXT: global_store_dwordx4 v245, v[8:11], s[0:1] offset:1056
-; CHECK-NEXT: global_store_dwordx4 v245, v[4:7], s[0:1] offset:1040
-; CHECK-NEXT: global_store_dwordx4 v245, a[12:15], s[0:1] offset:1024
+; CHECK-NEXT: v_add_f32_e32 v230, 1.0, v244
+; CHECK-NEXT: v_add_f32_e32 v231, 2.0, v228
+; CHECK-NEXT: global_store_dwordx2 v0, v[230:231], s[0:1]
+; CHECK-NEXT: global_store_dwordx4 v0, v[224:227], s[0:1] offset:1952
+; CHECK-NEXT: global_store_dwordx4 v0, v[232:235], s[0:1] offset:1936
+; CHECK-NEXT: global_store_dwordx4 v0, v[236:239], s[0:1] offset:1920
+; CHECK-NEXT: global_store_dwordx4 v0, v[192:195], s[0:1] offset:368
+; CHECK-NEXT: global_store_dwordx4 v0, v[196:199], s[0:1] offset:352
+; CHECK-NEXT: global_store_dwordx4 v0, v[200:203], s[0:1] offset:336
+; CHECK-NEXT: global_store_dwordx4 v0, v[204:207], s[0:1] offset:320
+; CHECK-NEXT: global_store_dwordx4 v0, v[208:211], s[0:1] offset:304
+; CHECK-NEXT: global_store_dwordx4 v0, v[212:215], s[0:1] offset:288
+; CHECK-NEXT: global_store_dwordx4 v0, v[216:219], s[0:1] offset:272
+; CHECK-NEXT: global_store_dwordx4 v0, v[220:223], s[0:1] offset:256
+; CHECK-NEXT: global_store_dwordx4 v0, v[160:163], s[0:1] offset:496
+; CHECK-NEXT: global_store_dwordx4 v0, v[164:167], s[0:1] offset:480
+; CHECK-NEXT: global_store_dwordx4 v0, v[168:171], s[0:1] offset:464
+; CHECK-NEXT: global_store_dwordx4 v0, v[172:175], s[0:1] offset:448
+; CHECK-NEXT: global_store_dwordx4 v0, v[176:179], s[0:1] offset:432
+; CHECK-NEXT: global_store_dwordx4 v0, v[180:183], s[0:1] offset:416
+; CHECK-NEXT: global_store_dwordx4 v0, v[184:187], s[0:1] offset:400
+; CHECK-NEXT: global_store_dwordx4 v0, v[188:191], s[0:1] offset:384
+; CHECK-NEXT: global_store_dwordx4 v0, v[128:131], s[0:1] offset:624
+; CHECK-NEXT: global_store_dwordx4 v0, v[132:135], s[0:1] offset:608
+; CHECK-NEXT: global_store_dwordx4 v0, v[136:139], s[0:1] offset:592
+; CHECK-NEXT: global_store_dwordx4 v0, v[140:143], s[0:1] offset:576
+; CHECK-NEXT: global_store_dwordx4 v0, v[144:147], s[0:1] offset:560
+; CHECK-NEXT: global_store_dwordx4 v0, v[148:151], s[0:1] offset:544
+; CHECK-NEXT: global_store_dwordx4 v0, v[152:155], s[0:1] offset:528
+; CHECK-NEXT: global_store_dwordx4 v0, v[156:159], s[0:1] offset:512
+; CHECK-NEXT: global_store_dwordx4 v0, v[96:99], s[0:1] offset:752
+; CHECK-NEXT: global_store_dwordx4 v0, v[100:103], s[0:1] offset:736
+; CHECK-NEXT: global_store_dwordx4 v0, v[104:107], s[0:1] offset:720
+; CHECK-NEXT: global_store_dwordx4 v0, v[108:111], s[0:1] offset:704
+; CHECK-NEXT: global_store_dwordx4 v0, v[112:115], s[0:1] offset:688
+; CHECK-NEXT: global_store_dwordx4 v0, v[116:119], s[0:1] offset:672
+; CHECK-NEXT: global_store_dwordx4 v0, v[120:123], s[0:1] offset:656
+; CHECK-NEXT: global_store_dwordx4 v0, v[124:127], s[0:1] offset:640
+; CHECK-NEXT: global_store_dwordx4 v0, v[64:67], s[0:1] offset:880
+; CHECK-NEXT: global_store_dwordx4 v0, v[68:71], s[0:1] offset:864
+; CHECK-NEXT: global_store_dwordx4 v0, v[72:75], s[0:1] offset:848
+; CHECK-NEXT: global_store_dwordx4 v0, v[76:79], s[0:1] offset:832
+; CHECK-NEXT: global_store_dwordx4 v0, v[80:83], s[0:1] offset:816
+; CHECK-NEXT: global_store_dwordx4 v0, v[84:87], s[0:1] offset:800
+; CHECK-NEXT: global_store_dwordx4 v0, v[88:91], s[0:1] offset:784
+; CHECK-NEXT: global_store_dwordx4 v0, v[92:95], s[0:1] offset:768
+; CHECK-NEXT: global_store_dwordx4 v0, v[32:35], s[0:1] offset:1008
+; CHECK-NEXT: global_store_dwordx4 v0, v[36:39], s[0:1] offset:992
+; CHECK-NEXT: global_store_dwordx4 v0, v[40:43], s[0:1] offset:976
+; CHECK-NEXT: global_store_dwordx4 v0, v[44:47], s[0:1] offset:960
+; CHECK-NEXT: global_store_dwordx4 v0, v[48:51], s[0:1] offset:944
+; CHECK-NEXT: global_store_dwordx4 v0, v[52:55], s[0:1] offset:928
+; CHECK-NEXT: global_store_dwordx4 v0, v[56:59], s[0:1] offset:912
+; CHECK-NEXT: global_store_dwordx4 v0, v[60:63], s[0:1] offset:896
+; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:1136
+; CHECK-NEXT: global_store_dwordx4 v0, v[4:7], s[0:1] offset:1120
+; CHECK-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] offset:1104
+; CHECK-NEXT: global_store_dwordx4 v0, v[12:15], s[0:1] offset:1088
+; CHECK-NEXT: global_store_dwordx4 v0, v[16:19], s[0:1] offset:1072
+; CHECK-NEXT: global_store_dwordx4 v0, v[20:23], s[0:1] offset:1056
+; CHECK-NEXT: global_store_dwordx4 v0, v[24:27], s[0:1] offset:1040
+; CHECK-NEXT: global_store_dwordx4 v0, v[28:31], s[0:1] offset:1024
; CHECK-NEXT: s_endpgm
; CHECK-NEXT: .Ltmp2:
; Loop body: MFMAs in AGPR form.
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 8e4832f7f9e6d..352fb414a62e5 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -510,14 +510,13 @@ define void @test_rewrite_mfma_subreg_insert1(float %arg0, float %arg1, ptr addr
; CHECK-LABEL: test_rewrite_mfma_subreg_insert1:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: global_load_dwordx4 v[2:5], v[2:3], off
+; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
+; CHECK-NEXT: v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[0:3]
; CHECK-NEXT: s_nop 3
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a0, v0
-; CHECK-NEXT: v_accvgpr_write_b32 a1, v1
+; CHECK-NEXT: v_accvgpr_mov_b32 a0, a1
+; CHECK-NEXT: v_accvgpr_mov_b32 a1, a2
+; CHECK-NEXT: v_accvgpr_mov_b32 a2, a3
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; use a[0:7]
; CHECK-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 63cd340f45657..56a5ffa2f0197 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -595,18 +595,20 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s2
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -615,14 +617,13 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -633,7 +634,6 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -683,65 +683,67 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -750,14 +752,12 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -769,21 +769,23 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s2, s0
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s2, s0
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s3, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -792,14 +794,13 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -810,7 +811,6 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -865,65 +865,67 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -932,14 +934,12 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -947,75 +947,80 @@ define amdgpu_ps float @ps_main(i32 %idx) {
;
; GFX11-FLATSCR-LABEL: ps_main:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v21
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v19, 0xbefcd89f :: v_dual_mov_b32 v30, v27
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v23, v21
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_mov_b32 v27, 0x3f523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v11, 0xbf3d349e
+; GFX11-FLATSCR-NEXT: v_and_b32_e32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, v27
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v13, 0x3efcd89f :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -1599,18 +1604,20 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s2
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -1619,14 +1626,13 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -1637,7 +1643,6 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -1687,65 +1692,67 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -1754,14 +1761,12 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -1773,21 +1778,23 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s2, s0
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s2, s0
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s3, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -1796,14 +1803,13 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -1814,7 +1820,6 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -1869,65 +1874,67 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -1936,14 +1943,12 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -1951,75 +1956,80 @@ define amdgpu_vs float @vs_main(i32 %idx) {
;
; GFX11-FLATSCR-LABEL: vs_main:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v21
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v19, 0xbefcd89f :: v_dual_mov_b32 v30, v27
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v23, v21
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_mov_b32 v27, 0x3f523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v11, 0xbf3d349e
+; GFX11-FLATSCR-NEXT: v_and_b32_e32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, v27
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v13, 0x3efcd89f :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -2601,85 +2611,85 @@ define amdgpu_cs float @cs_main(i32 %idx) {
;
; GFX9-FLATSCR-LABEL: cs_main:
; GFX9-FLATSCR: ; %bb.0:
+; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s2
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v9, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3f5f2ee2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3f523be1
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3f3d349e
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbeae29dc
; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0x3f5f2ee2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0x3f523be1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:320
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbeae29dc
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, v2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v10, 0x3eae29dc
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0x3eae29d8
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0x3e319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3e31934f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, 0xbf3d349e
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, 0xbf20e7f5
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0x3efcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, 0x3efcd89c
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0xbf5f2ee3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be3
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v31, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f638e37
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v22, v21
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xb702e758
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v5, 0xb7043519
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0xbe31934f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0xbe319356
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:240
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[15:18], s0 offset:224
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0xbf638e39
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, v26
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, 0x3f20e7f5
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, v15
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v29
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0x3703c499
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:272
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:256
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:208
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:192
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, v31
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, v11
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, v12
-; GFX9-FLATSCR-NEXT: scratch_load_dword v32, v4, off
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, v7
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], s0 offset:720
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf523be1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3f3d349c
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v5, v2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, v15
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, v10
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v10, 0xbf20e7f4
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, v8
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, v7
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, 0x3eae29dc
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0x3eae29d8
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, 0x3e319356
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0x3e31934f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf3d349e
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf20e7f5
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v23, 0x3efcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v24, 0x3efcd89c
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0xbf5f2ee3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v5, 0x3f638e37
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:288
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:320
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[16:19], s0 offset:240
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v10, 0xb702e758
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xb7043519
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbe31934f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0xbe319356
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:224
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0xbf638e39
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v24, 0x3f20e7f5
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, v21
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, v30
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0x3703c499
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[5:8], s0 offset:304
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:256
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[27:30], s0 offset:208
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], s0 offset:192
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v9
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, v26
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v30, v1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:784
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v22, v16
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0xbf5f2ee2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:736
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 offset:816
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, v23
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v21
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[27:30], s0 offset:768
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:832
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:752
-; GFX9-FLATSCR-NEXT: v_add_u32_e32 v0, 0x200, v31
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:800
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[15:18], s0 offset:704
-; GFX9-FLATSCR-NEXT: scratch_load_dword v0, v0, off
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v18
+; GFX9-FLATSCR-NEXT: scratch_load_dword v32, v1, off
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, v13
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], s0 offset:720
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0x3f20e7f4
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v10, 0xbf523be1
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3f3d349c
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, v7
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, v21
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, v8
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, v16
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v23
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, v5
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v30, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v31, v6
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], s0 offset:832
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:784
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v22
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf5f2ee2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], s0 offset:736
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:816
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v23, v2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v24, v0
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[28:31], s0 offset:768
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:752
+; GFX9-FLATSCR-NEXT: v_add_u32_e32 v5, 0x200, v9
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], s0 offset:800
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:704
+; GFX9-FLATSCR-NEXT: scratch_load_dword v0, v5, off
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: v_add_f32_e32 v0, v32, v0
; GFX9-FLATSCR-NEXT: ; return to shader part epilog
@@ -2691,65 +2701,67 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -2758,14 +2770,12 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -2776,87 +2786,87 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: s_getpc_b64 s[2:3]
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s2, s0
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x10
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0x3f5f2ee2
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0x3f3d349e
+; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v9, 0x1fc, v0
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3f523be1
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s2, s0
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3f3d349e
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s3, 0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, 0xbeae29dc
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v2
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, 0x3eae29dc
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0x3eae29d8
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0x3e319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3e31934f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, 0xbf3d349e
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, 0xbf20e7f5
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0x3efcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, 0x3efcd89c
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0xbf5f2ee3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0xbf523be3
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v31, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f638e37
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, v21
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xb702e758
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, 0xb7043519
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0xbe31934f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:240
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[15:18], s0 offset:224
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0xbf638e39
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, v26
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, 0x3f20e7f5
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, v15
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v29
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0x3703c499
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:272
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:256
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:208
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:192
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v31
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, v11
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v12
-; GFX9-FLATSCR-PAL-NEXT: scratch_load_dword v32, v4, off
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v7
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], s0 offset:720
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3f3d349c
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v2
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v15
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v10
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, 0xbf20e7f4
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, v8
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, v7
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, 0x3eae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0x3eae29d8
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, 0x3e319356
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0x3e31934f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, 0xbf3d349e
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf20e7f5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0x3efcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, 0x3efcd89c
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0xbf5f2ee3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, 0x3f638e37
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:288
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[16:19], s0 offset:240
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, 0xb702e758
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xb7043519
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbe31934f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0xbe319356
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:224
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0xbf638e39
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, 0x3f20e7f5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v21
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v30
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0x3703c499
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[5:8], s0 offset:304
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:256
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[27:30], s0 offset:208
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], s0 offset:192
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v9
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, v26
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, v1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:784
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, v16
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0xbf5f2ee2
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v21
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:736
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], s0 offset:816
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, v23
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v21
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[27:30], s0 offset:768
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:832
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], s0 offset:752
-; GFX9-FLATSCR-PAL-NEXT: v_add_u32_e32 v0, 0x200, v31
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:800
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[15:18], s0 offset:704
-; GFX9-FLATSCR-PAL-NEXT: scratch_load_dword v0, v0, off
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v18
+; GFX9-FLATSCR-PAL-NEXT: scratch_load_dword v32, v1, off
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v13
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], s0 offset:720
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0x3f20e7f4
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, 0xbf523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3f3d349c
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, v7
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, v21
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v8
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, v16
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v23
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, v5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v6
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], s0 offset:832
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:784
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v22
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], s0 offset:736
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:816
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, v2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v0
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[28:31], s0 offset:768
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[10:13], s0 offset:752
+; GFX9-FLATSCR-PAL-NEXT: v_add_u32_e32 v5, 0x200, v9
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], s0 offset:800
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[21:24], s0 offset:704
+; GFX9-FLATSCR-PAL-NEXT: scratch_load_dword v0, v5, off
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v32, v0
; GFX9-FLATSCR-PAL-NEXT: ; return to shader part epilog
@@ -2873,65 +2883,67 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -2940,14 +2952,12 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -2955,75 +2965,80 @@ define amdgpu_cs float @cs_main(i32 %idx) {
;
; GFX11-FLATSCR-LABEL: cs_main:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v21
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v19, 0xbefcd89f :: v_dual_mov_b32 v30, v27
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v23, v21
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_mov_b32 v27, 0x3f523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v11, 0xbf3d349e
+; GFX11-FLATSCR-NEXT: v_and_b32_e32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, v27
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v13, 0x3efcd89f :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -3604,18 +3619,20 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s5
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -3624,14 +3641,13 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -3642,7 +3658,6 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -3692,65 +3707,67 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -3759,14 +3776,12 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -3778,21 +3793,23 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, s8
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s0, s5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -3801,14 +3818,13 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -3819,7 +3835,6 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -3874,65 +3889,67 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -3941,14 +3958,12 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -3956,75 +3971,80 @@ define amdgpu_hs float @hs_main(i32 %idx) {
;
; GFX11-FLATSCR-LABEL: hs_main:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v21
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v19, 0xbefcd89f :: v_dual_mov_b32 v30, v27
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v23, v21
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_mov_b32 v27, 0x3f523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v11, 0xbf3d349e
+; GFX11-FLATSCR-NEXT: v_and_b32_e32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, v27
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v13, 0x3efcd89f :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -4605,18 +4625,20 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s5
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -4625,14 +4647,13 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -4643,7 +4664,6 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -4693,65 +4713,67 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -4760,14 +4782,12 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -4779,21 +4799,23 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, s8
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s0, s5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -4802,14 +4824,13 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -4820,7 +4841,6 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -4875,65 +4895,67 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -4942,14 +4964,12 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -4957,75 +4977,80 @@ define amdgpu_gs float @gs_main(i32 %idx) {
;
; GFX11-FLATSCR-LABEL: gs_main:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v21
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v19, 0xbefcd89f :: v_dual_mov_b32 v30, v27
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v23, v21
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_mov_b32 v27, 0x3f523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v11, 0xbf3d349e
+; GFX11-FLATSCR-NEXT: v_and_b32_e32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, v27
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v13, 0x3efcd89f :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -5615,18 +5640,20 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s5
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -5635,14 +5662,13 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -5653,7 +5679,6 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -5704,65 +5729,67 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -5771,15 +5798,13 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
+; GFX10-FLATSCR-NEXT: s_mov_b32 s2, s7
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_mov_b32 s2, s7
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -5791,21 +5816,23 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, s8
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s0, s5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -5814,14 +5841,13 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -5832,7 +5858,6 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -5888,65 +5913,67 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -5955,15 +5982,13 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
+; GFX10-FLATSCR-PAL-NEXT: s_mov_b32 s2, s5
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_mov_b32 s2, s5
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -5971,74 +5996,77 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
;
; GFX11-FLATSCR-LABEL: hs_ir_uses_scratch_offset:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, v28 :: v_dual_mov_b32 v19, 0xbefcd89f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v30, v27
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, 0xbf523be3 :: v_dual_mov_b32 v23, v21
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v17, 0xbeae29dc :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v27, 0x3f523be1 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, 0x3f638e37
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, v27 :: v_dual_mov_b32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v12, 0xbf20e7f5 :: v_dual_mov_b32 v23, v21
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_and_b32 v33, 0x1fc, v0
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v3, 0xbe319356
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_mov_b32 s2, s5
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
-; GFX11-FLATSCR-NEXT: s_mov_b32 s2, s5
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
@@ -6626,18 +6654,20 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR: ; %bb.0:
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s0, s5
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
-; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-NEXT: v_and_b32_e32 v30, 0x1fc, v0
+; GFX9-FLATSCR-NEXT: s_mov_b32 s0, 0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -6646,14 +6676,13 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -6664,7 +6693,6 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -6715,65 +6743,67 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v27
@@ -6782,15 +6812,13 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
+; GFX10-FLATSCR-NEXT: s_mov_b32 s2, s7
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-NEXT: s_mov_b32 s2, s7
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
@@ -6802,21 +6830,23 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, s8
; GFX9-FLATSCR-PAL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX9-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX9-FLATSCR-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-PAL-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-FLATSCR-PAL-NEXT: s_add_u32 flat_scratch_lo, s0, s5
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX9-FLATSCR-PAL-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
+; GFX9-FLATSCR-PAL-NEXT: v_and_b32_e32 v30, 0x1fc, v0
; GFX9-FLATSCR-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xbf20e7f4
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, v29
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v28
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v27
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
@@ -6825,14 +6855,13 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
+; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:320
+; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
@@ -6843,7 +6872,6 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX9-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], s0 offset:304
-; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], s0 offset:272
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], s0 offset:240
; GFX9-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], s0 offset:208
@@ -6899,65 +6927,67 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-FLATSCR-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-FLATSCR-PAL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xbf20e7f4
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v27, 0x3f5f2ee2
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v29, 0x3f3d349e
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v28, 0x3f523be1
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
; GFX10-FLATSCR-PAL-NEXT: v_and_b32_e32 v0, 0x1fc, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:320
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, v27
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, 0x3f638e37
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v4, 0xbf20e7f4
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v29
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, v27
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, 0xbefcd8a3
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v17
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v22, 0xbf5f2ee3
; GFX10-FLATSCR-PAL-NEXT: v_add_nc_u32_e32 v34, 0x200, v0
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[1:4], off offset:288
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v35, v0
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v1, 0xb7043519
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v3, 0xbe319356
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, 0xbf523be3
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[26:29], off offset:304
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:288
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[17:20], off offset:272
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v23, 0xbf638e39
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v22
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[4:7], off offset:320
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:224
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, 0x3f20e7f5
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v15, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v25
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[6:9], off offset:240
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[22:25], off offset:208
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
-; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v5, off
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v7, 0x3703c499
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
-; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v9, v6
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v10, v13
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v18, v12
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:192
+; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v1, v35, off
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v20, v17
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v21, v3
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v30, 0xbf523be1
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, 0x3f3d349c
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v31, v28
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, v11
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v16, v29
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v24, v26
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v25, v22
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v26, v27
+; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v0, 0x3f20e7f4
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:784
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v32, 0xbf5f2ee2
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v33, v27
@@ -6966,15 +6996,13 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[13:16], off offset:736
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v13, v19
; GFX10-FLATSCR-PAL-NEXT: v_mov_b32_e32 v14, v17
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[7:10], off offset:816
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[18:21], off offset:800
+; GFX10-FLATSCR-PAL-NEXT: s_mov_b32 s2, s5
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[23:26], off offset:768
+; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[30:33], off offset:752
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[3:6], off offset:720
; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[11:14], off offset:704
-; GFX10-FLATSCR-PAL-NEXT: s_mov_b32 s2, s5
-; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:832
; GFX10-FLATSCR-PAL-NEXT: scratch_load_dword v0, v34, off
; GFX10-FLATSCR-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-PAL-NEXT: v_add_f32_e32 v0, v1, v0
@@ -6982,74 +7010,77 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
;
; GFX11-FLATSCR-LABEL: gs_ir_uses_scratch_offset:
; GFX11-FLATSCR: ; %bb.0:
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xbf20e7f4 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v28, 0x3f3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v27, 0x3f523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v26, 0x3f5f2ee2
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:320
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, v28 :: v_dual_mov_b32 v19, 0xbefcd89f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v30, v27
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, 0xbf523be3 :: v_dual_mov_b32 v23, v21
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, 0x3f638e37 :: v_dual_mov_b32 v4, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, v27
-; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v17, 0xbeae29dc :: v_dual_mov_b32 v16, v24
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v27, 0x3f523be1 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, 0x3f638e37
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v21, 0xbf5f2ee3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v4, 0xbf20e7f4 :: v_dual_mov_b32 v5, v28
+; GFX11-FLATSCR-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, v27 :: v_dual_mov_b32 v7, v26
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v19, 0xbefcd89f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, 0xbf523be3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v17, 0xbeae29dc
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v11, 0xbf3d349e
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v6, 0x3eae29dc
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v8, 0x3e319356
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v9, 0x3e31934f
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v12, 0xbf20e7f5 :: v_dual_mov_b32 v23, v21
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[25:28], off offset:304
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[1:4], off offset:288
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v12, 0xbf20e7f5
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, 0x3efcd89f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v3, 0xbe319356
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:288
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v6, 0x3eae29dc :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3eae29d8
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v24 :: v_dual_and_b32 v33, 0x1fc, v0
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v3, 0xbe319356
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, 0x3efcd89c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v30, v27
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[6:9], off offset:240
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:224
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v14, 0x3f20e7f5 :: v_dual_mov_b32 v9, v6
-; GFX11-FLATSCR-NEXT: s_clause 0x3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v10, v13
+; GFX11-FLATSCR-NEXT: s_clause 0x4
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[17:20], off offset:272
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:320
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v25, v26
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v18, v12
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v20, v17
; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:752
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:736
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v5, v8 :: v_dual_mov_b32 v14, v17
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v14, v17
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v24, v21 :: v_dual_mov_b32 v21, v3
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v0, 0x3f20e7f4 :: v_dual_mov_b32 v5, v8
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v13, v19
-; GFX11-FLATSCR-NEXT: s_clause 0x4
+; GFX11-FLATSCR-NEXT: s_mov_b32 s2, s5
+; GFX11-FLATSCR-NEXT: s_clause 0x2
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[7:10], off offset:816
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[18:21], off offset:800
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[22:25], off offset:768
+; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FLATSCR-NEXT: s_clause 0x2
+; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:832
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[3:6], off offset:720
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[11:14], off offset:704
-; GFX11-FLATSCR-NEXT: scratch_load_b32 v1, v33, off offset:512
-; GFX11-FLATSCR-NEXT: s_mov_b32 s2, s5
+; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off offset:512
; GFX11-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FLATSCR-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11-FLATSCR-NEXT: ; return to shader part epilog
%v1 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
%v2 = extractelement <81 x float> <float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
index 03aeb6c57f871..c63c2e784dbc7 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
@@ -629,107 +629,108 @@ define amdgpu_kernel void @v_shl_v32i64(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-NEXT: v_lshlrev_b32_e32 v24, 8, v0
+; GFX1251-NEXT: v_lshlrev_b32_e32 v80, 8, v0
; GFX1251-NEXT: s_wait_kmcnt 0x0
-; GFX1251-NEXT: s_clause 0x13
-; GFX1251-NEXT: global_load_b128 v[0:3], v24, s[2:3] offset:400
-; GFX1251-NEXT: global_load_b128 v[4:7], v24, s[2:3] offset:144
-; GFX1251-NEXT: global_load_b128 v[8:11], v24, s[2:3] offset:128
-; GFX1251-NEXT: global_load_b128 v[12:15], v24, s[2:3] offset:384
-; GFX1251-NEXT: global_load_b128 v[16:19], v24, s[2:3] offset:432
-; GFX1251-NEXT: global_load_b128 v[20:23], v24, s[2:3] offset:176
-; GFX1251-NEXT: global_load_b128 v[26:29], v24, s[2:3] offset:160
-; GFX1251-NEXT: global_load_b128 v[30:33], v24, s[2:3] offset:416
-; GFX1251-NEXT: global_load_b128 v[34:37], v24, s[2:3] offset:464
-; GFX1251-NEXT: global_load_b128 v[38:41], v24, s[2:3] offset:208
-; GFX1251-NEXT: global_load_b128 v[42:45], v24, s[2:3] offset:192
-; GFX1251-NEXT: global_load_b128 v[46:49], v24, s[2:3] offset:448
-; GFX1251-NEXT: global_load_b128 v[50:53], v24, s[2:3] offset:496
-; GFX1251-NEXT: global_load_b128 v[54:57], v24, s[2:3] offset:240
-; GFX1251-NEXT: global_load_b128 v[58:61], v24, s[2:3] offset:224
-; GFX1251-NEXT: global_load_b128 v[62:65], v24, s[2:3] offset:480
-; GFX1251-NEXT: global_load_b128 v[66:69], v24, s[2:3] offset:16
-; GFX1251-NEXT: global_load_b128 v[70:73], v24, s[2:3]
-; GFX1251-NEXT: global_load_b128 v[74:77], v24, s[2:3] offset:272
-; GFX1251-NEXT: global_load_b128 v[78:81], v24, s[2:3] offset:256
-; GFX1251-NEXT: s_wait_loadcnt 0x12
+; GFX1251-NEXT: s_clause 0x11
+; GFX1251-NEXT: global_load_b128 v[0:3], v80, s[2:3] offset:400
+; GFX1251-NEXT: global_load_b128 v[4:7], v80, s[2:3] offset:144
+; GFX1251-NEXT: global_load_b128 v[8:11], v80, s[2:3] offset:128
+; GFX1251-NEXT: global_load_b128 v[12:15], v80, s[2:3] offset:384
+; GFX1251-NEXT: global_load_b128 v[16:19], v80, s[2:3] offset:432
+; GFX1251-NEXT: global_load_b128 v[20:23], v80, s[2:3] offset:176
+; GFX1251-NEXT: global_load_b128 v[24:27], v80, s[2:3] offset:160
+; GFX1251-NEXT: global_load_b128 v[28:31], v80, s[2:3] offset:416
+; GFX1251-NEXT: global_load_b128 v[32:35], v80, s[2:3] offset:464
+; GFX1251-NEXT: global_load_b128 v[36:39], v80, s[2:3] offset:208
+; GFX1251-NEXT: global_load_b128 v[40:43], v80, s[2:3] offset:192
+; GFX1251-NEXT: global_load_b128 v[44:47], v80, s[2:3] offset:448
+; GFX1251-NEXT: global_load_b128 v[48:51], v80, s[2:3] offset:496
+; GFX1251-NEXT: global_load_b128 v[52:55], v80, s[2:3] offset:240
+; GFX1251-NEXT: global_load_b128 v[56:59], v80, s[2:3] offset:224
+; GFX1251-NEXT: global_load_b128 v[60:63], v80, s[2:3] offset:480
+; GFX1251-NEXT: global_load_b128 v[64:67], v80, s[2:3] offset:16
+; GFX1251-NEXT: global_load_b128 v[68:71], v80, s[2:3]
+; GFX1251-NEXT: s_wait_loadcnt 0x10
; GFX1251-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[6:7]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[4:5]
-; GFX1251-NEXT: s_wait_loadcnt 0x10
+; GFX1251-NEXT: s_wait_loadcnt 0xe
; GFX1251-NEXT: v_lshlrev_b64_e32 v[6:7], v14, v[10:11]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[8:9]
-; GFX1251-NEXT: s_wait_loadcnt 0xe
+; GFX1251-NEXT: s_wait_loadcnt 0xc
; GFX1251-NEXT: v_lshlrev_b64_e32 v[10:11], v18, v[22:23]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[8:9], v16, v[20:21]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[12:15], v80, s[2:3] offset:272
+; GFX1251-NEXT: global_load_b128 v[16:19], v80, s[2:3] offset:256
; GFX1251-NEXT: s_wait_loadcnt 0xc
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[14:15], v32, v[28:29]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[12:13], v30, v[26:27]
-; GFX1251-NEXT: global_load_b128 v[26:29], v24, s[2:3] offset:48
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[22:23], v30, v[26:27]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[20:21], v28, v[24:25]
+; GFX1251-NEXT: global_load_b128 v[24:27], v80, s[2:3] offset:48
; GFX1251-NEXT: s_wait_loadcnt 0xb
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[18:19], v36, v[40:41]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[16:17], v34, v[38:39]
-; GFX1251-NEXT: global_load_b128 v[30:33], v24, s[2:3] offset:304
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[30:31], v34, v[38:39]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[28:29], v32, v[36:37]
+; GFX1251-NEXT: global_load_b128 v[32:35], v80, s[2:3] offset:304
; GFX1251-NEXT: s_wait_loadcnt 0xa
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[22:23], v48, v[44:45]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[20:21], v46, v[42:43]
-; GFX1251-NEXT: s_clause 0x1
-; GFX1251-NEXT: global_load_b128 v[34:37], v24, s[2:3] offset:80
-; GFX1251-NEXT: global_load_b128 v[42:45], v24, s[2:3] offset:336
-; GFX1251-NEXT: s_wait_loadcnt 0xa
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[40:41], v52, v[56:57]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[38:39], v50, v[54:55]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[38:39], v46, v[42:43]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[36:37], v44, v[40:41]
+; GFX1251-NEXT: global_load_b128 v[40:43], v80, s[2:3] offset:80
+; GFX1251-NEXT: s_wait_loadcnt 0x9
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[46:47], v50, v[54:55]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[44:45], v48, v[52:53]
+; GFX1251-NEXT: global_load_b128 v[48:51], v80, s[2:3] offset:336
; GFX1251-NEXT: s_wait_loadcnt 0x8
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[48:49], v64, v[60:61]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[46:47], v62, v[58:59]
-; GFX1251-NEXT: global_load_b128 v[50:53], v24, s[2:3] offset:64
-; GFX1251-NEXT: s_wait_loadcnt 0x6
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[56:57], v76, v[68:69]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[54:55], v74, v[66:67]
-; GFX1251-NEXT: global_load_b128 v[58:61], v24, s[2:3] offset:32
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[54:55], v62, v[58:59]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[52:53], v60, v[56:57]
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[56:59], v80, s[2:3] offset:64
+; GFX1251-NEXT: global_load_b128 v[60:63], v80, s[2:3] offset:32
+; GFX1251-NEXT: s_wait_loadcnt 0x7
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[14:15], v14, v[66:67]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[12:13], v12, v[64:65]
; GFX1251-NEXT: s_wait_loadcnt 0x6
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[64:65], v80, v[72:73]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[62:63], v78, v[70:71]
-; GFX1251-NEXT: global_load_b128 v[66:69], v24, s[2:3] offset:288
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[18:19], v18, v[70:71]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[16:17], v16, v[68:69]
+; GFX1251-NEXT: global_load_b128 v[64:67], v80, s[2:3] offset:288
; GFX1251-NEXT: s_wait_loadcnt 0x5
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[28:29], v32, v[28:29]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[26:27], v30, v[26:27]
-; GFX1251-NEXT: global_load_b128 v[30:33], v24, s[2:3] offset:320
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[26:27], v34, v[26:27]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[24:25], v32, v[24:25]
+; GFX1251-NEXT: global_load_b128 v[32:35], v80, s[2:3] offset:320
; GFX1251-NEXT: s_wait_loadcnt 0x4
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[36:37], v44, v[36:37]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[34:35], v42, v[34:35]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[42:43], v50, v[42:43]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[40:41], v48, v[40:41]
; GFX1251-NEXT: s_clause 0x3
-; GFX1251-NEXT: global_load_b128 v[42:45], v24, s[2:3] offset:368
-; GFX1251-NEXT: global_load_b128 v[70:73], v24, s[2:3] offset:112
-; GFX1251-NEXT: global_load_b128 v[74:77], v24, s[2:3] offset:96
-; GFX1251-NEXT: global_load_b128 v[78:81], v24, s[2:3] offset:352
+; GFX1251-NEXT: global_load_b128 v[48:51], v80, s[2:3] offset:368
+; GFX1251-NEXT: global_load_b128 v[68:71], v80, s[2:3] offset:112
+; GFX1251-NEXT: global_load_b128 v[72:75], v80, s[2:3] offset:96
+; GFX1251-NEXT: global_load_b128 v[76:79], v80, s[2:3] offset:352
; GFX1251-NEXT: s_wait_loadcnt 0x4
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[32:33], v32, v[52:53]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[30:31], v30, v[50:51]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[34:35], v34, v[58:59]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[32:33], v32, v[56:57]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[58:59], v66, v[62:63]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[56:57], v64, v[60:61]
; GFX1251-NEXT: s_wait_loadcnt 0x2
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[44:45], v44, v[72:73]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[42:43], v42, v[70:71]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[50:51], v50, v[70:71]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[48:49], v48, v[68:69]
; GFX1251-NEXT: s_wait_loadcnt 0x0
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[72:73], v80, v[76:77]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[70:71], v78, v[74:75]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[52:53], v68, v[60:61]
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[50:51], v66, v[58:59]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[68:69], v76, v[72:73]
; GFX1251-NEXT: s_clause 0xf
-; GFX1251-NEXT: global_store_b128 v24, v[70:73], s[0:1] offset:96
-; GFX1251-NEXT: global_store_b128 v24, v[42:45], s[0:1] offset:112
-; GFX1251-NEXT: global_store_b128 v24, v[30:33], s[0:1] offset:64
-; GFX1251-NEXT: global_store_b128 v24, v[34:37], s[0:1] offset:80
-; GFX1251-NEXT: global_store_b128 v24, v[50:53], s[0:1] offset:32
-; GFX1251-NEXT: global_store_b128 v24, v[26:29], s[0:1] offset:48
-; GFX1251-NEXT: global_store_b128 v24, v[62:65], s[0:1]
-; GFX1251-NEXT: global_store_b128 v24, v[54:57], s[0:1] offset:16
-; GFX1251-NEXT: global_store_b128 v24, v[46:49], s[0:1] offset:224
-; GFX1251-NEXT: global_store_b128 v24, v[38:41], s[0:1] offset:240
-; GFX1251-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:192
-; GFX1251-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:208
-; GFX1251-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:160
-; GFX1251-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:176
-; GFX1251-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:128
-; GFX1251-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:144
+; GFX1251-NEXT: global_store_b128 v80, v[68:71], s[0:1] offset:96
+; GFX1251-NEXT: global_store_b128 v80, v[48:51], s[0:1] offset:112
+; GFX1251-NEXT: global_store_b128 v80, v[32:35], s[0:1] offset:64
+; GFX1251-NEXT: global_store_b128 v80, v[40:43], s[0:1] offset:80
+; GFX1251-NEXT: global_store_b128 v80, v[56:59], s[0:1] offset:32
+; GFX1251-NEXT: global_store_b128 v80, v[24:27], s[0:1] offset:48
+; GFX1251-NEXT: global_store_b128 v80, v[16:19], s[0:1]
+; GFX1251-NEXT: global_store_b128 v80, v[12:15], s[0:1] offset:16
+; GFX1251-NEXT: global_store_b128 v80, v[52:55], s[0:1] offset:224
+; GFX1251-NEXT: global_store_b128 v80, v[44:47], s[0:1] offset:240
+; GFX1251-NEXT: global_store_b128 v80, v[36:39], s[0:1] offset:192
+; GFX1251-NEXT: global_store_b128 v80, v[28:31], s[0:1] offset:208
+; GFX1251-NEXT: global_store_b128 v80, v[20:23], s[0:1] offset:160
+; GFX1251-NEXT: global_store_b128 v80, v[8:11], s[0:1] offset:176
+; GFX1251-NEXT: global_store_b128 v80, v[4:7], s[0:1] offset:128
+; GFX1251-NEXT: global_store_b128 v80, v[0:3], s[0:1] offset:144
; GFX1251-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX1251-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
index 38b8e4026a8ef..dad0bb7c25961 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
@@ -387,29 +387,30 @@ define void @shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy(ptr addrspace(
; GFX90A-LABEL: shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4, v5, v6, v7
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v0, v5
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v5
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, v6
+; GFX90A-NEXT: v_mov_b32_e32 v8, v7
+; GFX90A-NEXT: v_mov_b32_e32 v6, v5
+; GFX90A-NEXT: v_mov_b32_e32 v7, v5
+; GFX90A-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: v_mov_b32_e32 v8, 0
; GFX940-NEXT: ;;#ASMSTART
; GFX940-NEXT: ; def v4, v5, v6, v7
; GFX940-NEXT: ;;#ASMEND
-; GFX940-NEXT: s_nop 0
-; GFX940-NEXT: v_mov_b32_e32 v0, v5
-; GFX940-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[1,0]
-; GFX940-NEXT: v_mov_b32_e32 v1, v5
-; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX940-NEXT: v_mov_b32_e32 v0, 0
+; GFX940-NEXT: v_mov_b32_e32 v9, v6
+; GFX940-NEXT: v_mov_b32_e32 v8, v7
+; GFX940-NEXT: v_mov_b32_e32 v6, v5
+; GFX940-NEXT: v_mov_b32_e32 v7, v5
+; GFX940-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1]
; GFX940-NEXT: s_waitcnt vmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
index 73af3fdf237c8..98b6480f8b8f9 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
@@ -538,9 +538,8 @@ define void @v_shuffle_v2f32_v4f32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -551,9 +550,8 @@ define void @v_shuffle_v2f32_v4f32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -879,9 +877,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +889,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1501,8 +1497,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1509,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3067,8 +3061,7 @@ define void @v_shuffle_v2f32_v4f32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -3080,8 +3073,7 @@ define void @v_shuffle_v2f32_v4f32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v8f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v8f32.ll
index b87360c0b00fe..5c6abe4f1458f 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v8f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v8f32.ll
@@ -1246,8 +1246,7 @@ define void @v_shuffle_v2f32_v8f32__15_12(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v7
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1259,8 +1258,7 @@ define void @v_shuffle_v2f32_v8f32__15_12(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v7
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1673,9 +1671,8 @@ define void @v_shuffle_v2f32_v8f32__5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1686,9 +1683,8 @@ define void @v_shuffle_v2f32_v8f32__5_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x float> asm "; def $0", "=v"()
@@ -4699,8 +4695,7 @@ define void @v_shuffle_v2f32_v8f32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v7
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -4712,8 +4707,7 @@ define void @v_shuffle_v2f32_v8f32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v7
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6085,8 +6079,7 @@ define void @v_shuffle_v2f32_v8f32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v5
-; GFX90A-NEXT: v_mov_b32_e32 v1, v6
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6098,8 +6091,7 @@ define void @v_shuffle_v2f32_v8f32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v5
-; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7817,9 +7809,8 @@ define void @v_shuffle_v2f32_v8f32__13_8(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7830,9 +7821,8 @@ define void @v_shuffle_v2f32_v8f32__13_8(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x float> asm "; def $0", "=v"()
@@ -12415,8 +12405,7 @@ define void @v_shuffle_v2f32_v8f32__13_14(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v5
-; GFX90A-NEXT: v_mov_b32_e32 v1, v6
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -12428,8 +12417,7 @@ define void @v_shuffle_v2f32_v8f32__13_14(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v5
-; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
index 23132713a076e..47cc3b77d6035 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
@@ -538,9 +538,8 @@ define void @v_shuffle_v2i32_v4i32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -551,9 +550,8 @@ define void @v_shuffle_v2i32_v4i32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -879,9 +877,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +889,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1501,8 +1497,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1509,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3067,8 +3061,7 @@ define void @v_shuffle_v2i32_v4i32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -3080,8 +3073,7 @@ define void @v_shuffle_v2i32_v4i32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
index ee8f880b6ab50..cbf8bc3d1fa7d 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
@@ -1673,9 +1673,8 @@ define void @v_shuffle_v2i32_v8i32__5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1686,9 +1685,8 @@ define void @v_shuffle_v2i32_v8i32__5_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x i32> asm "; def $0", "=v"()
@@ -7817,9 +7815,8 @@ define void @v_shuffle_v2i32_v8i32__13_8(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7830,9 +7827,8 @@ define void @v_shuffle_v2i32_v8i32__13_8(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x i32> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
index e9431b78d127c..2e29108efaec9 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
@@ -538,9 +538,8 @@ define void @v_shuffle_v2p3_v4p3__7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -551,9 +550,8 @@ define void @v_shuffle_v2p3_v4p3__7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -879,9 +877,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +889,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1501,8 +1497,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1509,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3067,8 +3061,7 @@ define void @v_shuffle_v2p3_v4p3__5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -3080,8 +3073,7 @@ define void @v_shuffle_v2p3_v4p3__5_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll
index 6d08104469221..c677c62b77690 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll
@@ -1673,9 +1673,8 @@ define void @v_shuffle_v2p3_v8p3__5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1686,9 +1685,8 @@ define void @v_shuffle_v2p3_v8p3__5_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -7817,9 +7815,8 @@ define void @v_shuffle_v2p3_v8p3__13_8(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v5
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7830,9 +7827,8 @@ define void @v_shuffle_v2p3_v8p3__13_8(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x ptr addrspace(3)> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
index 1a61bff240e52..ebb9c6a6e32b1 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll
@@ -167,14 +167,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -182,15 +183,15 @@ define void @v_shuffle_v4f32_v2f32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -561,15 +562,16 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -577,15 +579,16 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -668,26 +671,31 @@ define void @v_shuffle_v4f32_v2f32__3_3_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v2f32__3_3_3_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v1
+; GFX90A-NEXT: v_mov_b32_e32 v6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: global_store_dwordx4 v2, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v2f32__3_3_3_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v1
+; GFX942-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: global_store_dwordx4 v2, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -861,7 +869,8 @@ define void @v_shuffle_v4f32_v2f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -874,7 +883,8 @@ define void @v_shuffle_v4f32_v2f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -957,7 +967,8 @@ define void @v_shuffle_v4f32_v2f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -973,7 +984,8 @@ define void @v_shuffle_v4f32_v2f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1119,13 +1131,14 @@ define void @v_shuffle_v4f32_v2f32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1135,13 +1148,14 @@ define void @v_shuffle_v4f32_v2f32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -1281,15 +1295,16 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1297,15 +1312,16 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -1695,7 +1711,8 @@ define void @v_shuffle_v4f32_v2f32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1711,7 +1728,8 @@ define void @v_shuffle_v4f32_v2f32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1746,13 +1764,14 @@ define void @v_shuffle_v4f32_v2f32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1762,13 +1781,14 @@ define void @v_shuffle_v4f32_v2f32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2118,7 +2138,8 @@ define void @v_shuffle_v4f32_v2f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2131,7 +2152,8 @@ define void @v_shuffle_v4f32_v2f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2210,15 +2232,16 @@ define void @v_shuffle_v4f32_v2f32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2226,15 +2249,16 @@ define void @v_shuffle_v4f32_v2f32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2475,32 +2499,37 @@ define void @v_shuffle_v4f32_v2f32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v2f32__3_3_1_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v2f32__3_3_1_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2783,15 +2812,16 @@ define void @v_shuffle_v4f32_v2f32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2799,15 +2829,16 @@ define void @v_shuffle_v4f32_v2f32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x float> asm "; def $0", "=v"()
@@ -2894,7 +2925,8 @@ define void @v_shuffle_v4f32_v2f32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2907,7 +2939,8 @@ define void @v_shuffle_v4f32_v2f32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
index 1a12ef01a6f63..019649a8001ad 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll
@@ -1249,7 +1249,8 @@ define void @v_shuffle_v4f32_v3f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1262,7 +1263,8 @@ define void @v_shuffle_v4f32_v3f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1394,7 +1396,8 @@ define void @v_shuffle_v4f32_v3f32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1410,8 +1413,8 @@ define void @v_shuffle_v4f32_v3f32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1879,32 +1882,35 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v3f32__5_5_1_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v3f32__5_5_1_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -2050,33 +2056,35 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v3f32__5_5_4_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v3f32__5_5_4_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -3141,7 +3149,8 @@ define void @v_shuffle_v4f32_v3f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3154,7 +3163,8 @@ define void @v_shuffle_v4f32_v3f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3286,7 +3296,8 @@ define void @v_shuffle_v4f32_v3f32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3302,8 +3313,8 @@ define void @v_shuffle_v4f32_v3f32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3826,15 +3837,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3842,16 +3854,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -3939,15 +3951,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3955,16 +3968,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -4137,7 +4150,8 @@ define void @v_shuffle_v4f32_v3f32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4150,7 +4164,8 @@ define void @v_shuffle_v4f32_v3f32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4653,15 +4668,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4669,16 +4685,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -4766,8 +4782,9 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -4780,8 +4797,9 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5866,7 +5884,8 @@ define void @v_shuffle_v4f32_v3f32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5883,7 +5902,8 @@ define void @v_shuffle_v4f32_v3f32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6019,7 +6039,8 @@ define void @v_shuffle_v4f32_v3f32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6032,7 +6053,8 @@ define void @v_shuffle_v4f32_v3f32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6485,15 +6507,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -6501,16 +6524,16 @@ define void @v_shuffle_v4f32_v3f32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x float> asm "; def $0", "=v"()
@@ -6648,8 +6671,9 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6662,8 +6686,9 @@ define void @v_shuffle_v4f32_v3f32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
index 6089a077c89fb..f7c722b63580a 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll
@@ -331,12 +331,13 @@ define void @v_shuffle_v4f32_v4f32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -351,8 +352,9 @@ define void @v_shuffle_v4f32_v4f32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -439,7 +441,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -454,8 +457,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -538,8 +541,9 @@ define void @v_shuffle_v4f32_v4f32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -550,8 +554,9 @@ define void @v_shuffle_v4f32_v4f32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -624,7 +629,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -636,7 +642,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1108,15 +1115,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1124,15 +1132,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1218,15 +1227,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1234,16 +1244,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1331,8 +1341,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1344,8 +1355,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1423,26 +1435,31 @@ define void @v_shuffle_v4f32_v4f32__7_7_7_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_7_7_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v4f32__7_7_7_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1616,7 +1633,8 @@ define void @v_shuffle_v4f32_v4f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1629,7 +1647,8 @@ define void @v_shuffle_v4f32_v4f32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1710,7 +1729,8 @@ define void @v_shuffle_v4f32_v4f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1723,7 +1743,8 @@ define void @v_shuffle_v4f32_v4f32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1806,7 +1827,8 @@ define void @v_shuffle_v4f32_v4f32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1822,8 +1844,8 @@ define void @v_shuffle_v4f32_v4f32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1917,7 +1939,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1933,8 +1956,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2084,7 +2107,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2100,8 +2124,9 @@ define void @v_shuffle_v4f32_v4f32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2196,7 +2221,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2213,7 +2239,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2309,7 +2336,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2326,7 +2354,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2467,15 +2496,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2483,15 +2513,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2579,15 +2610,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v9
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2595,15 +2627,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v9
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -2691,15 +2724,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2707,15 +2741,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -3313,7 +3348,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3329,8 +3365,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3369,7 +3405,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3385,8 +3422,9 @@ define void @v_shuffle_v4f32_v4f32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3481,7 +3519,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3498,7 +3537,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3594,7 +3634,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3611,7 +3652,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4170,7 +4212,8 @@ define void @v_shuffle_v4f32_v4f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4183,7 +4226,8 @@ define void @v_shuffle_v4f32_v4f32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4264,7 +4308,8 @@ define void @v_shuffle_v4f32_v4f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4277,7 +4322,8 @@ define void @v_shuffle_v4f32_v4f32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4360,7 +4406,8 @@ define void @v_shuffle_v4f32_v4f32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4376,8 +4423,8 @@ define void @v_shuffle_v4f32_v4f32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4471,7 +4518,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4487,8 +4535,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4574,15 +4622,17 @@ define void @v_shuffle_v4f32_v4f32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v2
+; GFX90A-NEXT: v_mov_b32_e32 v7, v2
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4590,15 +4640,17 @@ define void @v_shuffle_v4f32_v4f32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -4746,7 +4798,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4763,7 +4816,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4858,7 +4912,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4875,7 +4930,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5076,8 +5132,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -5092,8 +5149,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5125,15 +5183,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5141,15 +5200,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5237,15 +5297,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5253,16 +5314,16 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5851,15 +5912,17 @@ define void @v_shuffle_v4f32_v4f32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5867,15 +5930,17 @@ define void @v_shuffle_v4f32_v4f32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -5965,7 +6030,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5981,8 +6047,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6021,7 +6087,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6038,7 +6105,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6133,7 +6201,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6150,7 +6219,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6807,7 +6877,8 @@ define void @v_shuffle_v4f32_v4f32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6820,7 +6891,8 @@ define void @v_shuffle_v4f32_v4f32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6903,7 +6975,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6916,7 +6989,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6995,15 +7069,16 @@ define void @v_shuffle_v4f32_v4f32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7011,16 +7086,16 @@ define void @v_shuffle_v4f32_v4f32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -7113,7 +7188,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7129,8 +7205,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7268,7 +7344,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7281,7 +7358,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7470,8 +7548,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7486,8 +7565,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7581,8 +7661,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7597,9 +7678,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7632,8 +7713,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7645,8 +7727,9 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -8241,15 +8324,16 @@ define void @v_shuffle_v4f32_v4f32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -8257,16 +8341,16 @@ define void @v_shuffle_v4f32_v4f32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -8359,7 +8443,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8375,8 +8460,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8465,7 +8550,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8478,7 +8564,8 @@ define void @v_shuffle_v4f32_v4f32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8512,7 +8599,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8525,7 +8613,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9086,7 +9175,8 @@ define void @v_shuffle_v4f32_v4f32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9103,7 +9193,8 @@ define void @v_shuffle_v4f32_v4f32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9199,7 +9290,8 @@ define void @v_shuffle_v4f32_v4f32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9215,8 +9307,8 @@ define void @v_shuffle_v4f32_v4f32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9295,7 +9387,8 @@ define void @v_shuffle_v4f32_v4f32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9308,7 +9401,8 @@ define void @v_shuffle_v4f32_v4f32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9391,7 +9485,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9404,7 +9499,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9489,7 +9585,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9506,7 +9603,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9603,7 +9701,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9619,8 +9718,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9707,26 +9806,31 @@ define void @v_shuffle_v4f32_v4f32__7_4_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_4_6_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v2
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v4f32__7_4_6_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -9959,15 +10063,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v8, v5
+; GFX90A-NEXT: v_mov_b32_e32 v9, v5
+; GFX90A-NEXT: v_mov_b32_e32 v10, v1
+; GFX90A-NEXT: v_mov_b32_e32 v11, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[8:11], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -9975,15 +10081,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v10, v1
+; GFX942-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[8:11], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -10072,15 +10180,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -10088,16 +10198,17 @@ define void @v_shuffle_v4f32_v4f32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v7
+; GFX942-NEXT: v_mov_b32_e32 v3, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -10174,26 +10285,31 @@ define void @v_shuffle_v4f32_v4f32__7_7_5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_7_5_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v4f32__7_7_5_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -10698,7 +10814,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, v5
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10715,7 +10832,8 @@ define void @v_shuffle_v4f32_v4f32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10812,7 +10930,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v6, v7
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10828,8 +10947,8 @@ define void @v_shuffle_v4f32_v4f32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v6, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10916,26 +11035,31 @@ define void @v_shuffle_v4f32_v4f32__7_4_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4f32_v4f32__7_4_7_7:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v3
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4f32_v4f32__7_4_7_7:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v3
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -11014,7 +11138,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -11027,7 +11152,8 @@ define void @v_shuffle_v4f32_v4f32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
index 47ba90e5c0dfa..f069e14b1e21e 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll
@@ -167,14 +167,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -182,15 +183,15 @@ define void @v_shuffle_v4i32_v2i32__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -561,15 +562,16 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -577,15 +579,16 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -668,26 +671,31 @@ define void @v_shuffle_v4i32_v2i32__3_3_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v2i32__3_3_3_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v1
+; GFX90A-NEXT: v_mov_b32_e32 v6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: global_store_dwordx4 v2, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v2i32__3_3_3_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v1
+; GFX942-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: global_store_dwordx4 v2, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -864,7 +872,8 @@ define void @v_shuffle_v4i32_v2i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -877,7 +886,8 @@ define void @v_shuffle_v4i32_v2i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -963,7 +973,8 @@ define void @v_shuffle_v4i32_v2i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -979,7 +990,8 @@ define void @v_shuffle_v4i32_v2i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1125,13 +1137,14 @@ define void @v_shuffle_v4i32_v2i32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1141,13 +1154,14 @@ define void @v_shuffle_v4i32_v2i32__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -1287,15 +1301,16 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1303,15 +1318,16 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -1701,7 +1717,8 @@ define void @v_shuffle_v4i32_v2i32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1717,7 +1734,8 @@ define void @v_shuffle_v4i32_v2i32__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1752,13 +1770,14 @@ define void @v_shuffle_v4i32_v2i32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1768,13 +1787,14 @@ define void @v_shuffle_v4i32_v2i32__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2124,7 +2144,8 @@ define void @v_shuffle_v4i32_v2i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2137,7 +2158,8 @@ define void @v_shuffle_v4i32_v2i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2216,15 +2238,16 @@ define void @v_shuffle_v4i32_v2i32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2232,15 +2255,16 @@ define void @v_shuffle_v4i32_v2i32__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2481,32 +2505,37 @@ define void @v_shuffle_v4i32_v2i32__3_3_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v2i32__3_3_1_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v2i32__3_3_1_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2789,15 +2818,16 @@ define void @v_shuffle_v4i32_v2i32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2805,15 +2835,16 @@ define void @v_shuffle_v4i32_v2i32__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x i32> asm "; def $0", "=v"()
@@ -2900,7 +2931,8 @@ define void @v_shuffle_v4i32_v2i32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2913,7 +2945,8 @@ define void @v_shuffle_v4i32_v2i32__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
index 3eb951c294ebd..4035b3a1a17c8 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll
@@ -1249,7 +1249,8 @@ define void @v_shuffle_v4i32_v3i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1262,7 +1263,8 @@ define void @v_shuffle_v4i32_v3i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1394,7 +1396,8 @@ define void @v_shuffle_v4i32_v3i32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1410,8 +1413,8 @@ define void @v_shuffle_v4i32_v3i32__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1879,32 +1882,35 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v3i32__5_5_1_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v3i32__5_5_1_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -2050,33 +2056,35 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v3i32__5_5_4_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v3i32__5_5_4_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -3141,7 +3149,8 @@ define void @v_shuffle_v4i32_v3i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3154,7 +3163,8 @@ define void @v_shuffle_v4i32_v3i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3286,7 +3296,8 @@ define void @v_shuffle_v4i32_v3i32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3302,8 +3313,8 @@ define void @v_shuffle_v4i32_v3i32__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3826,15 +3837,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3842,16 +3854,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -3939,15 +3951,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3955,16 +3968,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -4137,7 +4150,8 @@ define void @v_shuffle_v4i32_v3i32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4150,7 +4164,8 @@ define void @v_shuffle_v4i32_v3i32__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4653,15 +4668,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4669,16 +4685,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -4766,8 +4782,9 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -4780,8 +4797,9 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5866,7 +5884,8 @@ define void @v_shuffle_v4i32_v3i32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5883,7 +5902,8 @@ define void @v_shuffle_v4i32_v3i32__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6019,7 +6039,8 @@ define void @v_shuffle_v4i32_v3i32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6032,7 +6053,8 @@ define void @v_shuffle_v4i32_v3i32__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6485,15 +6507,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -6501,16 +6524,16 @@ define void @v_shuffle_v4i32_v3i32__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x i32> asm "; def $0", "=v"()
@@ -6648,8 +6671,9 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6662,8 +6686,9 @@ define void @v_shuffle_v4i32_v3i32__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
index 7c97ebc04e1bb..afd5cf3f31a68 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll
@@ -331,12 +331,13 @@ define void @v_shuffle_v4i32_v4i32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -351,8 +352,9 @@ define void @v_shuffle_v4i32_v4i32__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -439,7 +441,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -454,8 +457,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -538,8 +541,9 @@ define void @v_shuffle_v4i32_v4i32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -550,8 +554,9 @@ define void @v_shuffle_v4i32_v4i32__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -624,7 +629,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -636,7 +642,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1108,15 +1115,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1124,15 +1132,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1218,15 +1227,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1234,16 +1244,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1331,8 +1341,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1344,8 +1355,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1423,26 +1435,31 @@ define void @v_shuffle_v4i32_v4i32__7_7_7_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_7_7_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v4i32__7_7_7_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1616,7 +1633,8 @@ define void @v_shuffle_v4i32_v4i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1629,7 +1647,8 @@ define void @v_shuffle_v4i32_v4i32__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1710,7 +1729,8 @@ define void @v_shuffle_v4i32_v4i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1723,7 +1743,8 @@ define void @v_shuffle_v4i32_v4i32__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1806,7 +1827,8 @@ define void @v_shuffle_v4i32_v4i32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1822,8 +1844,8 @@ define void @v_shuffle_v4i32_v4i32__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1917,7 +1939,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1933,8 +1956,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2084,7 +2107,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2100,8 +2124,9 @@ define void @v_shuffle_v4i32_v4i32__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2196,7 +2221,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2213,7 +2239,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2309,7 +2336,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2326,7 +2354,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2467,15 +2496,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2483,15 +2513,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2579,15 +2610,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v9
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2595,15 +2627,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v9
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -2691,15 +2724,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2707,15 +2741,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -3313,7 +3348,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3329,8 +3365,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3369,7 +3405,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3385,8 +3422,9 @@ define void @v_shuffle_v4i32_v4i32__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3481,7 +3519,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3498,7 +3537,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3594,7 +3634,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3611,7 +3652,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4170,7 +4212,8 @@ define void @v_shuffle_v4i32_v4i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4183,7 +4226,8 @@ define void @v_shuffle_v4i32_v4i32__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4264,7 +4308,8 @@ define void @v_shuffle_v4i32_v4i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4277,7 +4322,8 @@ define void @v_shuffle_v4i32_v4i32__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4360,7 +4406,8 @@ define void @v_shuffle_v4i32_v4i32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4376,8 +4423,8 @@ define void @v_shuffle_v4i32_v4i32__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4471,7 +4518,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4487,8 +4535,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4574,15 +4622,17 @@ define void @v_shuffle_v4i32_v4i32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v2
+; GFX90A-NEXT: v_mov_b32_e32 v7, v2
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4590,15 +4640,17 @@ define void @v_shuffle_v4i32_v4i32__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -4746,7 +4798,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4763,7 +4816,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4858,7 +4912,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4875,7 +4930,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5076,8 +5132,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -5092,8 +5149,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5125,15 +5183,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5141,15 +5200,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5237,15 +5297,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5253,16 +5314,16 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5851,15 +5912,17 @@ define void @v_shuffle_v4i32_v4i32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5867,15 +5930,17 @@ define void @v_shuffle_v4i32_v4i32__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -5965,7 +6030,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5981,8 +6047,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6021,7 +6087,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6038,7 +6105,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6133,7 +6201,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6150,7 +6219,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6807,7 +6877,8 @@ define void @v_shuffle_v4i32_v4i32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6820,7 +6891,8 @@ define void @v_shuffle_v4i32_v4i32__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6903,7 +6975,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6916,7 +6989,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6995,15 +7069,16 @@ define void @v_shuffle_v4i32_v4i32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7011,16 +7086,16 @@ define void @v_shuffle_v4i32_v4i32__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -7113,7 +7188,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7129,8 +7205,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7268,7 +7344,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7281,7 +7358,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7470,8 +7548,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7486,8 +7565,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7581,8 +7661,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7597,9 +7678,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7632,8 +7713,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7645,8 +7727,9 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -8241,15 +8324,16 @@ define void @v_shuffle_v4i32_v4i32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -8257,16 +8341,16 @@ define void @v_shuffle_v4i32_v4i32__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -8359,7 +8443,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8375,8 +8460,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8465,7 +8550,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8478,7 +8564,8 @@ define void @v_shuffle_v4i32_v4i32__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8512,7 +8599,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8525,7 +8613,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9086,7 +9175,8 @@ define void @v_shuffle_v4i32_v4i32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9103,7 +9193,8 @@ define void @v_shuffle_v4i32_v4i32__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9199,7 +9290,8 @@ define void @v_shuffle_v4i32_v4i32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9215,8 +9307,8 @@ define void @v_shuffle_v4i32_v4i32__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9295,7 +9387,8 @@ define void @v_shuffle_v4i32_v4i32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9308,7 +9401,8 @@ define void @v_shuffle_v4i32_v4i32__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9391,7 +9485,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9404,7 +9499,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9489,7 +9585,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9506,7 +9603,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9603,7 +9701,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9619,8 +9718,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9707,26 +9806,31 @@ define void @v_shuffle_v4i32_v4i32__7_4_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_4_6_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v2
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v4i32__7_4_6_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -9959,15 +10063,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v8, v5
+; GFX90A-NEXT: v_mov_b32_e32 v9, v5
+; GFX90A-NEXT: v_mov_b32_e32 v10, v1
+; GFX90A-NEXT: v_mov_b32_e32 v11, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[8:11], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -9975,15 +10081,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v10, v1
+; GFX942-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[8:11], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -10072,15 +10180,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -10088,16 +10198,17 @@ define void @v_shuffle_v4i32_v4i32__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v7
+; GFX942-NEXT: v_mov_b32_e32 v3, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -10174,26 +10285,31 @@ define void @v_shuffle_v4i32_v4i32__7_7_5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_7_5_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v4i32__7_7_5_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -10698,7 +10814,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, v5
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10715,7 +10832,8 @@ define void @v_shuffle_v4i32_v4i32__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10812,7 +10930,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v6, v7
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10828,8 +10947,8 @@ define void @v_shuffle_v4i32_v4i32__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v6, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10916,26 +11035,31 @@ define void @v_shuffle_v4i32_v4i32__7_4_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4i32_v4i32__7_4_7_7:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v3
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4i32_v4i32__7_4_7_7:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v3
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -11014,7 +11138,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -11027,7 +11152,8 @@ define void @v_shuffle_v4i32_v4i32__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
index 06dfe92932f1d..dea005d0d1ad7 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll
@@ -167,14 +167,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -182,15 +183,15 @@ define void @v_shuffle_v4p3_v2p3__3_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -561,15 +562,16 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -577,15 +579,16 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -668,26 +671,31 @@ define void @v_shuffle_v4p3_v2p3__3_3_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v2p3__3_3_3_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v1
+; GFX90A-NEXT: v_mov_b32_e32 v6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: global_store_dwordx4 v2, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v2p3__3_3_3_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v1
+; GFX942-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: global_store_dwordx4 v2, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -864,7 +872,8 @@ define void @v_shuffle_v4p3_v2p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -877,7 +886,8 @@ define void @v_shuffle_v4p3_v2p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -963,7 +973,8 @@ define void @v_shuffle_v4p3_v2p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -979,7 +990,8 @@ define void @v_shuffle_v4p3_v2p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1125,13 +1137,14 @@ define void @v_shuffle_v4p3_v2p3__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1141,13 +1154,14 @@ define void @v_shuffle_v4p3_v2p3__3_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1287,15 +1301,16 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1303,15 +1318,16 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1701,7 +1717,8 @@ define void @v_shuffle_v4p3_v2p3__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1717,7 +1734,8 @@ define void @v_shuffle_v4p3_v2p3__3_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1752,13 +1770,14 @@ define void @v_shuffle_v4p3_v2p3__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1768,13 +1787,14 @@ define void @v_shuffle_v4p3_v2p3__3_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2124,7 +2144,8 @@ define void @v_shuffle_v4p3_v2p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2137,7 +2158,8 @@ define void @v_shuffle_v4p3_v2p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2216,15 +2238,16 @@ define void @v_shuffle_v4p3_v2p3__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2232,15 +2255,16 @@ define void @v_shuffle_v4p3_v2p3__3_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2481,32 +2505,37 @@ define void @v_shuffle_v4p3_v2p3__3_3_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v2p3__3_3_1_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v2p3__3_3_1_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[0:1]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2789,15 +2818,16 @@ define void @v_shuffle_v4p3_v2p3__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:1]
+; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2805,15 +2835,16 @@ define void @v_shuffle_v4p3_v2p3__3_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:1]
+; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:3]
+; GFX942-NEXT: ; def v[4:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <2 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2900,7 +2931,8 @@ define void @v_shuffle_v4p3_v2p3__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2913,7 +2945,8 @@ define void @v_shuffle_v4p3_v2p3__3_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
index c421e52f33e38..4e2aa2af7e664 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll
@@ -1249,7 +1249,8 @@ define void @v_shuffle_v4p3_v3p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1262,7 +1263,8 @@ define void @v_shuffle_v4p3_v3p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1394,7 +1396,8 @@ define void @v_shuffle_v4p3_v3p3__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1410,8 +1413,8 @@ define void @v_shuffle_v4p3_v3p3__4_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1879,32 +1882,35 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v3p3__5_5_1_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v3p3__5_5_1_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2050,33 +2056,35 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v3p3__5_5_4_0:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:6]
+; GFX90A-NEXT: ; def v[6:8]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v9, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v3p3__5_5_4_0:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:6]
+; GFX942-NEXT: ; def v[6:8]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v9, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3141,7 +3149,8 @@ define void @v_shuffle_v4p3_v3p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3154,7 +3163,8 @@ define void @v_shuffle_v4p3_v3p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3286,7 +3296,8 @@ define void @v_shuffle_v4p3_v3p3__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v7, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3302,8 +3313,8 @@ define void @v_shuffle_v4p3_v3p3__4_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3826,15 +3837,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3842,16 +3854,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3939,15 +3951,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -3955,16 +3968,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4137,7 +4150,8 @@ define void @v_shuffle_v4p3_v3p3__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4150,7 +4164,8 @@ define void @v_shuffle_v4p3_v3p3__4_3_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4653,15 +4668,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4669,16 +4685,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4766,8 +4782,9 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -4780,8 +4797,9 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5866,7 +5884,8 @@ define void @v_shuffle_v4p3_v3p3__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5883,7 +5902,8 @@ define void @v_shuffle_v4p3_v3p3__1_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6019,7 +6039,8 @@ define void @v_shuffle_v4p3_v3p3__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6032,7 +6053,8 @@ define void @v_shuffle_v4p3_v3p3__4_5_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6485,15 +6507,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: global_store_dwordx4 v7, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -6501,16 +6524,16 @@ define void @v_shuffle_v4p3_v3p3__5_5_1_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-NEXT: global_store_dwordx4 v7, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -6648,8 +6671,9 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6662,8 +6686,9 @@ define void @v_shuffle_v4p3_v3p3__5_5_4_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
index 7402b2b995491..b698d712a6278 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll
@@ -331,12 +331,13 @@ define void @v_shuffle_v4p3_v4p3__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -351,8 +352,9 @@ define void @v_shuffle_v4p3_v4p3__7_0_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -439,7 +441,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -454,8 +457,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -538,8 +541,9 @@ define void @v_shuffle_v4p3_v4p3__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-NEXT: global_store_dwordx4 v4, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -550,8 +554,9 @@ define void @v_shuffle_v4p3_v4p3__7_4_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: global_store_dwordx4 v4, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -624,7 +629,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -636,7 +642,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_u_u(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1108,15 +1115,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1124,15 +1132,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1218,15 +1227,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1234,16 +1244,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1331,8 +1341,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1344,8 +1355,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1423,26 +1435,31 @@ define void @v_shuffle_v4p3_v4p3__7_7_7_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_7_7_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v4p3__7_7_7_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1616,7 +1633,8 @@ define void @v_shuffle_v4p3_v4p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1629,7 +1647,8 @@ define void @v_shuffle_v4p3_v4p3__1_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1710,7 +1729,8 @@ define void @v_shuffle_v4p3_v4p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1723,7 +1743,8 @@ define void @v_shuffle_v4p3_v4p3__3_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1806,7 +1827,8 @@ define void @v_shuffle_v4p3_v4p3__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1822,8 +1844,8 @@ define void @v_shuffle_v4p3_v4p3__5_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1917,7 +1939,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1933,8 +1956,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -2084,7 +2107,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2100,8 +2124,9 @@ define void @v_shuffle_v4p3_v4p3__7_2_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2196,7 +2221,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2213,7 +2239,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2309,7 +2336,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -2326,7 +2354,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_0_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2467,15 +2496,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2483,15 +2513,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2579,15 +2610,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v9
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2595,15 +2627,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v9
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -2691,15 +2724,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v10, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -2707,15 +2741,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_0(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v10, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -3313,7 +3348,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3329,8 +3365,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -3369,7 +3405,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v9
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3385,8 +3422,9 @@ define void @v_shuffle_v4p3_v4p3__7_2_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, v9
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[4:5] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3481,7 +3519,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3498,7 +3537,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3594,7 +3634,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -3611,7 +3652,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_1_1(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4170,7 +4212,8 @@ define void @v_shuffle_v4p3_v4p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4183,7 +4226,8 @@ define void @v_shuffle_v4p3_v4p3__1_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4264,7 +4308,8 @@ define void @v_shuffle_v4p3_v4p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4277,7 +4322,8 @@ define void @v_shuffle_v4p3_v4p3__3_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4360,7 +4406,8 @@ define void @v_shuffle_v4p3_v4p3__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4376,8 +4423,8 @@ define void @v_shuffle_v4p3_v4p3__5_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4471,7 +4518,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4487,8 +4535,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -4574,15 +4622,17 @@ define void @v_shuffle_v4p3_v4p3__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v2
+; GFX90A-NEXT: v_mov_b32_e32 v7, v2
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -4590,15 +4640,17 @@ define void @v_shuffle_v4p3_v4p3__7_0_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -4746,7 +4798,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4763,7 +4816,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -4858,7 +4912,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -4875,7 +4930,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_2_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5076,8 +5132,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -5092,8 +5149,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[6:7] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5125,15 +5183,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5141,15 +5200,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5237,15 +5297,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5253,16 +5314,16 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_2(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5851,15 +5912,17 @@ define void @v_shuffle_v4p3_v4p3__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -5867,15 +5930,17 @@ define void @v_shuffle_v4p3_v4p3__7_0_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -5965,7 +6030,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5981,8 +6047,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6021,7 +6087,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6038,7 +6105,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6133,7 +6201,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6150,7 +6219,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_3_3(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -6807,7 +6877,8 @@ define void @v_shuffle_v4p3_v4p3__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6820,7 +6891,8 @@ define void @v_shuffle_v4p3_v4p3__5_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6903,7 +6975,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6916,7 +6989,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6995,15 +7069,16 @@ define void @v_shuffle_v4p3_v4p3__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -7011,16 +7086,16 @@ define void @v_shuffle_v4p3_v4p3__7_0_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -7113,7 +7188,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7129,8 +7205,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7268,7 +7344,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7281,7 +7358,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_4_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -7470,8 +7548,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7486,8 +7565,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7581,8 +7661,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v5
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7597,9 +7678,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -7632,8 +7713,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -7645,8 +7727,9 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -8241,15 +8324,16 @@ define void @v_shuffle_v4p3_v4p3__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[6:9]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
+; GFX90A-NEXT: global_store_dwordx4 v10, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -8257,16 +8341,16 @@ define void @v_shuffle_v4p3_v4p3__7_0_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[6:9]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX942-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -8359,7 +8443,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8375,8 +8460,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8465,7 +8550,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8478,7 +8564,8 @@ define void @v_shuffle_v4p3_v4p3__7_4_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -8512,7 +8599,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v5
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8525,7 +8613,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_5_5(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9086,7 +9175,8 @@ define void @v_shuffle_v4p3_v4p3__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9103,7 +9193,8 @@ define void @v_shuffle_v4p3_v4p3__1_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[4:5] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9199,7 +9290,8 @@ define void @v_shuffle_v4p3_v4p3__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v3
+; GFX90A-NEXT: v_mov_b32_e32 v5, v6
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9215,8 +9307,8 @@ define void @v_shuffle_v4p3_v4p3__3_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[6:7] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v5, v6
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9295,7 +9387,8 @@ define void @v_shuffle_v4p3_v4p3__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9308,7 +9401,8 @@ define void @v_shuffle_v4p3_v4p3__5_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9391,7 +9485,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9404,7 +9499,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9489,7 +9585,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, v4
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9506,7 +9603,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9603,7 +9701,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v7, v6
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9619,8 +9718,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_6_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9707,26 +9806,31 @@ define void @v_shuffle_v4p3_v4p3__7_4_6_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_4_6_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v2
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v4p3__7_4_6_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -9959,15 +10063,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:7]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v8, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v8, v5
+; GFX90A-NEXT: v_mov_b32_e32 v9, v5
+; GFX90A-NEXT: v_mov_b32_e32 v10, v1
+; GFX90A-NEXT: v_mov_b32_e32 v11, v4
+; GFX90A-NEXT: global_store_dwordx4 v6, v[8:11], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -9975,15 +10081,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_1_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[4:7]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v8, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v10, v1
+; GFX942-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-NEXT: global_store_dwordx4 v6, v[8:11], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -10072,15 +10180,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:5]
+; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:3]
+; GFX90A-NEXT: ; def v[4:7]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v7
+; GFX90A-NEXT: v_mov_b32_e32 v1, v7
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v3, v6
+; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -10088,16 +10198,17 @@ define void @v_shuffle_v4p3_v4p3__7_7_3_6(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:5]
+; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:3]
+; GFX942-NEXT: ; def v[4:7]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[4:5], v[2:3] op_sel:[1,0]
; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v7
+; GFX942-NEXT: v_mov_b32_e32 v3, v6
+; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -10174,26 +10285,31 @@ define void @v_shuffle_v4p3_v4p3__7_7_5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_7_5_6:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v3
+; GFX90A-NEXT: v_mov_b32_e32 v8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v9, v2
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v4p3__7_7_5_6:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v6, 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[2:3] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -10698,7 +10814,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[2:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v6, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v2, v5
+; GFX90A-NEXT: v_mov_b32_e32 v3, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, v5
; GFX90A-NEXT: global_store_dwordx4 v6, v[2:5], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10715,7 +10832,8 @@ define void @v_shuffle_v4p3_v4p3__7_0_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[2:5]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v2, v5
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v4, v5
; GFX942-NEXT: global_store_dwordx4 v6, v[2:5], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10812,7 +10930,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v4, v7
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
; GFX90A-NEXT: v_mov_b32_e32 v6, v7
; GFX90A-NEXT: global_store_dwordx4 v8, v[4:7], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -10828,8 +10947,8 @@ define void @v_shuffle_v4p3_v4p3__7_2_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v4, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v2
; GFX942-NEXT: v_mov_b32_e32 v6, v7
; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -10916,26 +11035,31 @@ define void @v_shuffle_v4p3_v4p3__7_4_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v4p3_v4p3__7_4_7_7:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v3
+; GFX90A-NEXT: global_store_dwordx4 v4, v[6:9], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: v_shuffle_v4p3_v4p3__7_4_7_7:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_mov_b32_e32 v9, v3
+; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -11014,7 +11138,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: v_mov_b32_e32 v2, v3
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -11027,7 +11152,8 @@ define void @v_shuffle_v4p3_v4p3__7_6_7_7(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
+; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: v_mov_b32_e32 v2, v3
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll b/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
index 626f547c31952..1612b1f301b74 100644
--- a/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
@@ -28,99 +28,97 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %
; CHECK-NEXT: s_mov_b32 s3, 6
; CHECK-NEXT: v_lshlrev_b64 v[2:3], s3, v[2:3]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], s[0:1], 0, v[2:3]
-; CHECK-NEXT: global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT: v_lshl_add_u64 v[6:7], s[0:1], 0, v[2:3]
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[6:7], off offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, v7
-; CHECK-NEXT: v_mov_b32_e32 v12, v6
-; CHECK-NEXT: v_mov_b32_e32 v13, v5
-; CHECK-NEXT: v_mov_b32_e32 v14, v4
-; CHECK-NEXT: global_load_dwordx4 v[4:7], v[2:3], off
+; CHECK-NEXT: v_mov_b32_e32 v1, v11
+; CHECK-NEXT: v_mov_b32_e32 v2, v10
+; CHECK-NEXT: v_mov_b32_e32 v3, v9
+; CHECK-NEXT: v_mov_b32_e32 v4, v8
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[6:7], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v15, v7
-; CHECK-NEXT: v_mov_b32_e32 v16, v6
-; CHECK-NEXT: v_mov_b32_e32 v17, v5
-; CHECK-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
-; CHECK-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v5, v17
-; CHECK-NEXT: v_mov_b32_e32 v6, v16
-; CHECK-NEXT: v_mov_b32_e32 v7, v15
-; CHECK-NEXT: v_mov_b32_e32 v8, v14
-; CHECK-NEXT: v_mov_b32_e32 v9, v13
-; CHECK-NEXT: v_mov_b32_e32 v10, v12
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v25, v11
-; CHECK-NEXT: v_mov_b32_e32 v26, v10
-; CHECK-NEXT: v_mov_b32_e32 v27, v9
-; CHECK-NEXT: v_mov_b32_e32 v28, v8
-; CHECK-NEXT: v_mov_b32_e32 v29, v7
-; CHECK-NEXT: v_mov_b32_e32 v30, v6
-; CHECK-NEXT: v_mov_b32_e32 v31, v5
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:32
+; CHECK-NEXT: v_mov_b32_e32 v5, v11
+; CHECK-NEXT: v_mov_b32_e32 v16, v10
+; CHECK-NEXT: v_mov_b32_e32 v17, v9
+; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 killed $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; CHECK-NEXT: v_mov_b32_e32 v9, v17
+; CHECK-NEXT: v_mov_b32_e32 v10, v16
+; CHECK-NEXT: v_mov_b32_e32 v11, v5
+; CHECK-NEXT: v_mov_b32_e32 v12, v4
+; CHECK-NEXT: v_mov_b32_e32 v13, v3
+; CHECK-NEXT: v_mov_b32_e32 v14, v2
+; CHECK-NEXT: v_mov_b32_e32 v15, v1
+; CHECK-NEXT: v_mov_b32_e32 v25, v15
+; CHECK-NEXT: v_mov_b32_e32 v26, v14
+; CHECK-NEXT: v_mov_b32_e32 v27, v13
+; CHECK-NEXT: v_mov_b32_e32 v28, v12
+; CHECK-NEXT: v_mov_b32_e32 v29, v11
+; CHECK-NEXT: v_mov_b32_e32 v30, v10
+; CHECK-NEXT: v_mov_b32_e32 v31, v9
+; CHECK-NEXT: v_mov_b32_e32 v2, v8
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[6:7], off offset:32
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, v13
-; CHECK-NEXT: v_mov_b32_e32 v6, v12
-; CHECK-NEXT: v_mov_b32_e32 v7, v11
-; CHECK-NEXT: ; kill: def $vgpr10 killed $vgpr10 killed $vgpr10_vgpr11_vgpr12_vgpr13 killed $exec
+; CHECK-NEXT: v_mov_b32_e32 v14, v11
+; CHECK-NEXT: v_mov_b32_e32 v15, v10
+; CHECK-NEXT: v_mov_b32_e32 v16, v9
+; CHECK-NEXT: v_mov_b32_e32 v4, v8
; CHECK-NEXT: s_mov_b64 s[6:7], 32
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, s[6:7]
-; CHECK-NEXT: global_load_dwordx4 v[12:15], v[2:3], off offset:16
+; CHECK-NEXT: v_lshl_add_u64 v[6:7], v[6:7], 0, s[6:7]
+; CHECK-NEXT: global_load_dwordx4 v[6:9], v[6:7], off offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, v15
-; CHECK-NEXT: v_mov_b32_e32 v2, v14
-; CHECK-NEXT: v_mov_b32_e32 v3, v13
-; CHECK-NEXT: v_mov_b32_e32 v4, v12
-; CHECK-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v11, v7
-; CHECK-NEXT: v_mov_b32_e32 v12, v6
-; CHECK-NEXT: v_mov_b32_e32 v13, v5
-; CHECK-NEXT: v_mov_b32_e32 v14, v4
-; CHECK-NEXT: v_mov_b32_e32 v15, v3
-; CHECK-NEXT: v_mov_b32_e32 v16, v2
+; CHECK-NEXT: v_mov_b32_e32 v1, v9
+; CHECK-NEXT: v_mov_b32_e32 v3, v8
+; CHECK-NEXT: v_mov_b32_e32 v12, v7
+; CHECK-NEXT: v_mov_b32_e32 v13, v6
+; CHECK-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
+; CHECK-NEXT: v_mov_b32_e32 v5, v16
+; CHECK-NEXT: v_mov_b32_e32 v6, v15
+; CHECK-NEXT: v_mov_b32_e32 v7, v14
+; CHECK-NEXT: v_mov_b32_e32 v8, v13
+; CHECK-NEXT: v_mov_b32_e32 v9, v12
+; CHECK-NEXT: v_mov_b32_e32 v10, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, v11
+; CHECK-NEXT: v_mov_b32_e32 v18, v10
+; CHECK-NEXT: v_mov_b32_e32 v19, v9
+; CHECK-NEXT: v_mov_b32_e32 v20, v8
+; CHECK-NEXT: v_mov_b32_e32 v21, v7
+; CHECK-NEXT: v_mov_b32_e32 v22, v6
+; CHECK-NEXT: v_mov_b32_e32 v23, v5
+; CHECK-NEXT: v_mov_b32_e32 v24, v4
+; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 killed $exec
+; CHECK-NEXT: v_mov_b32_e32 v3, v31
+; CHECK-NEXT: v_mov_b32_e32 v4, v30
+; CHECK-NEXT: v_mov_b32_e32 v5, v29
+; CHECK-NEXT: v_mov_b32_e32 v6, v28
+; CHECK-NEXT: v_mov_b32_e32 v7, v27
+; CHECK-NEXT: v_mov_b32_e32 v8, v26
+; CHECK-NEXT: v_mov_b32_e32 v9, v25
+; CHECK-NEXT: v_mov_b32_e32 v10, v24
+; CHECK-NEXT: v_mov_b32_e32 v11, v23
+; CHECK-NEXT: v_mov_b32_e32 v12, v22
+; CHECK-NEXT: v_mov_b32_e32 v13, v21
+; CHECK-NEXT: v_mov_b32_e32 v14, v20
+; CHECK-NEXT: v_mov_b32_e32 v15, v19
+; CHECK-NEXT: v_mov_b32_e32 v16, v18
; CHECK-NEXT: v_mov_b32_e32 v17, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, v17
-; CHECK-NEXT: v_mov_b32_e32 v2, v16
-; CHECK-NEXT: v_mov_b32_e32 v3, v15
-; CHECK-NEXT: v_mov_b32_e32 v4, v14
-; CHECK-NEXT: v_mov_b32_e32 v5, v13
-; CHECK-NEXT: v_mov_b32_e32 v6, v12
-; CHECK-NEXT: v_mov_b32_e32 v7, v11
-; CHECK-NEXT: v_mov_b32_e32 v24, v10
-; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v9, v31
-; CHECK-NEXT: v_mov_b32_e32 v10, v30
-; CHECK-NEXT: v_mov_b32_e32 v11, v29
-; CHECK-NEXT: v_mov_b32_e32 v12, v28
-; CHECK-NEXT: v_mov_b32_e32 v13, v27
-; CHECK-NEXT: v_mov_b32_e32 v14, v26
-; CHECK-NEXT: v_mov_b32_e32 v15, v25
-; CHECK-NEXT: v_mov_b32_e32 v16, v24
-; CHECK-NEXT: v_mov_b32_e32 v17, v7
-; CHECK-NEXT: v_mov_b32_e32 v18, v6
-; CHECK-NEXT: v_mov_b32_e32 v19, v5
-; CHECK-NEXT: v_mov_b32_e32 v20, v4
-; CHECK-NEXT: v_mov_b32_e32 v21, v3
-; CHECK-NEXT: v_mov_b32_e32 v22, v2
-; CHECK-NEXT: v_mov_b32_e32 v23, v1
-; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[16:17]
-; CHECK-NEXT: v_mov_b64_e32 v[4:5], s[18:19]
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], s[14:15]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], s[12:13]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[10:11]
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
+; CHECK-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; CHECK-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], s[14:15]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[12:13]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
+; CHECK-NEXT: v_mov_b64_e32 v[22:23], s[8:9]
; CHECK-NEXT: v_mov_b32_e32 v1, s2
; CHECK-NEXT: s_nop 1
-; CHECK-NEXT: v_smfmac_f32_32x32x32_f16 v[8:23], v[2:5], v[24:31], v1 cbsz:1 abid:2
+; CHECK-NEXT: v_smfmac_f32_32x32x32_f16 v[2:17], v[18:21], v[22:29], v1 cbsz:1 abid:2
; CHECK-NEXT: s_nop 11
-; CHECK-NEXT: v_mov_b32_e32 v1, v23
-; CHECK-NEXT: v_mov_b32_e32 v6, v22
-; CHECK-NEXT: v_mov_b32_e32 v7, v21
-; CHECK-NEXT: v_mov_b32_e32 v2, v20
-; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v3, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[16:17]
+; CHECK-NEXT: v_mov_b64_e32 v[22:23], v[14:15]
+; CHECK-NEXT: v_mov_b64_e32 v[20:21], v[12:13]
+; CHECK-NEXT: v_mov_b64_e32 v[18:19], v[10:11]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[24:25]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[22:23]
; CHECK-NEXT: s_mov_b32 s2, s0
; CHECK-NEXT: s_mov_b32 s3, s1
; CHECK-NEXT: s_mov_b32 s5, s6
@@ -129,33 +127,14 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %
; CHECK-NEXT: s_addc_u32 s4, s3, s4
; CHECK-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; CHECK-NEXT: s_mov_b32 s3, s4
-; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[2:3] offset:16
-; CHECK-NEXT: v_mov_b32_e32 v1, v19
-; CHECK-NEXT: v_mov_b32_e32 v6, v18
-; CHECK-NEXT: v_mov_b32_e32 v7, v17
-; CHECK-NEXT: v_mov_b32_e32 v2, v16
-; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v3, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mov_b32_e32 v5, v1
-; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32
-; CHECK-NEXT: v_mov_b32_e32 v1, v15
-; CHECK-NEXT: v_mov_b32_e32 v6, v14
-; CHECK-NEXT: v_mov_b32_e32 v7, v13
-; CHECK-NEXT: v_mov_b32_e32 v2, v12
-; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v3, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mov_b32_e32 v5, v1
-; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16
-; CHECK-NEXT: v_mov_b32_e32 v1, v11
-; CHECK-NEXT: v_mov_b32_e32 v6, v10
-; CHECK-NEXT: v_mov_b32_e32 v7, v9
-; CHECK-NEXT: v_mov_b32_e32 v2, v8
-; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v3, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: global_store_dwordx4 v0, v[26:29], s[2:3] offset:16
+; CHECK-NEXT: ; kill: def $vgpr18_vgpr19_vgpr20_vgpr21 killed $vgpr18_vgpr19_vgpr20_vgpr21 killed $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 killed $exec
+; CHECK-NEXT: global_store_dwordx4 v0, v[18:21], s[0:1] offset:32
+; CHECK-NEXT: ; kill: def $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9 killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9 killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 killed $exec
+; CHECK-NEXT: v_mov_b64_e32 v[12:13], v[8:9]
+; CHECK-NEXT: v_mov_b64_e32 v[10:11], v[6:7]
+; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[0:1] offset:16
+; CHECK-NEXT: ; kill: def $vgpr2_vgpr3_vgpr4_vgpr5 killed $vgpr2_vgpr3_vgpr4_vgpr5 killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9 killed $exec
; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; CHECK-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
index a2c5db0d2646d..efeac352b6025 100644
--- a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
@@ -15,14 +15,14 @@ define protected amdgpu_kernel void @test(ptr addrspace(1) %in, ptr addrspace(1)
; GFX942-NEXT: v_mov_b64_e32 v[12:13], v[2:3]
; GFX942-NEXT: v_mov_b64_e32 v[10:11], v[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, s4
-; GFX942-NEXT: v_mov_b32_e32 v9, s5
+; GFX942-NEXT: v_mov_b64_e32 v[14:15], s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v4, s6
; GFX942-NEXT: v_mov_b32_e32 v5, s7
; GFX942-NEXT: v_mov_b32_e32 v6, s7
; GFX942-NEXT: v_mov_b32_e32 v7, s7
+; GFX942-NEXT: v_mov_b32_e32 v8, s5
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_smfmac_i32_16x16x64_i8 v[10:13], v[8:9], v[4:7], v9
+; GFX942-NEXT: v_smfmac_i32_16x16x64_i8 v[10:13], v[14:15], v[4:7], v8
; GFX942-NEXT: s_nop 6
; GFX942-NEXT: global_store_dword v0, v13, s[2:3] offset:12
; GFX942-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 5c829d7ec88ef..e9783f3e1365f 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -35,10 +35,10 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: s_mov_b32 s43, 0xe8f000
; GFX6-NEXT: s_add_u32 s40, s40, s11
; GFX6-NEXT: s_addc_u32 s41, s41, 0
-; GFX6-NEXT: s_mov_b32 s2, 0x3fd00
-; GFX6-NEXT: s_mov_b64 s[8:9], 0x100
+; GFX6-NEXT: s_mov_b64 s[8:9], 0x80
; GFX6-NEXT: s_mov_b64 s[10:11], s[6:7]
-; GFX6-NEXT: s_mov_b64 s[12:13], 0x180
+; GFX6-NEXT: s_mov_b32 s2, 0x3fd00
+; GFX6-NEXT: s_mov_b64 s[12:13], 0x100
; GFX6-NEXT: s_mov_b64 s[14:15], s[6:7]
; GFX6-NEXT: s_mov_b64 s[16:17], 0x200
; GFX6-NEXT: s_mov_b64 s[18:19], s[6:7]
@@ -1835,15 +1835,14 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4076 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:4080 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:4080
-; GFX6-NEXT: s_mov_b64 s[4:5], 0x80
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3968
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3968
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3984
; GFX6-NEXT: s_mov_b32 s2, 0x40100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1851,7 +1850,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4000
; GFX6-NEXT: s_mov_b32 s2, 0x40500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1859,7 +1858,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4016
; GFX6-NEXT: s_mov_b32 s2, 0x40900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1867,15 +1866,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4016
-; GFX6-NEXT: s_mov_b32 s2, 0x40d00
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4032
; GFX6-NEXT: s_mov_b32 s2, 0x41100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1883,7 +1874,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4048
; GFX6-NEXT: s_mov_b32 s2, 0x41500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1891,7 +1882,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4064
; GFX6-NEXT: s_mov_b32 s2, 0x41900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1899,7 +1890,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4080
; GFX6-NEXT: s_mov_b32 s2, 0x41d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1907,7 +1898,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3968
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3968
; GFX6-NEXT: s_mov_b32 s2, 0x42100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1915,7 +1906,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3984
; GFX6-NEXT: s_mov_b32 s2, 0x42500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1923,7 +1914,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4000
; GFX6-NEXT: s_mov_b32 s2, 0x42900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1931,7 +1922,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4016
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4016
; GFX6-NEXT: s_mov_b32 s2, 0x42d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1939,7 +1930,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4032
; GFX6-NEXT: s_mov_b32 s2, 0x43100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1947,7 +1938,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4048
; GFX6-NEXT: s_mov_b32 s2, 0x43500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1955,7 +1946,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4064
; GFX6-NEXT: s_mov_b32 s2, 0x43900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1963,7 +1954,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4080
; GFX6-NEXT: s_mov_b32 s2, 0x43d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1971,7 +1962,16 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3968
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:4080
+; GFX6-NEXT: s_mov_b32 s2, 0x40d00
+; GFX6-NEXT: s_mov_b64 s[4:5], 0x180
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3968
; GFX6-NEXT: s_mov_b32 s2, 0x44100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1979,7 +1979,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3984
; GFX6-NEXT: s_mov_b32 s2, 0x44500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1987,7 +1987,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4000
; GFX6-NEXT: s_mov_b32 s2, 0x44900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -1995,7 +1995,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4016
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4016
; GFX6-NEXT: s_mov_b32 s2, 0x44d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2003,7 +2003,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4032
; GFX6-NEXT: s_mov_b32 s2, 0x45100
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2011,7 +2011,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4048
; GFX6-NEXT: s_mov_b32 s2, 0x45500
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2019,7 +2019,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4064
; GFX6-NEXT: s_mov_b32 s2, 0x45900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2027,7 +2027,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4080
; GFX6-NEXT: s_mov_b32 s2, 0x45d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2339,7 +2339,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4080
; GFX6-NEXT: s_mov_b32 s2, 0x4f900
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -2347,12 +2347,9 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4064
; GFX6-NEXT: ;;#ASMSTART
; GFX6-NEXT: ;;#ASMEND
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, s0, v5
-; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v4, vcc
; GFX6-NEXT: ;;#ASMSTART
; GFX6-NEXT: ;;#ASMEND
; GFX6-NEXT: ;;#ASMSTART
@@ -2365,16 +2362,17 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: ;;#ASMEND
; GFX6-NEXT: ;;#ASMSTART
; GFX6-NEXT: ;;#ASMEND
-; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4080
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s33 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s33 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s33 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s33 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s33 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s33 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v11, off, s[40:43], s33 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v12, off, s[40:43], s33 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, s0, v5
+; GFX6-NEXT: v_mov_b32_e32 v4, s1
+; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v4, vcc
; GFX6-NEXT: s_mov_b32 s33, 0x4f500
+; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7]
; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_store_dwordx4 v[9:12], v[7:8], s[36:39], 0 addr64 offset:4080
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[36:39], 0 addr64 offset:4064
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s33 ; 4-byte Folded Reload
@@ -2687,7 +2685,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x45900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4080
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2695,7 +2693,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x45500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4064
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2703,7 +2701,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x45100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4048
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2711,7 +2709,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x44d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4032
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2719,7 +2717,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x44900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4016
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4016
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2727,7 +2725,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x44500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4000
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
@@ -2735,143 +2733,143 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b32 s16, 0x44100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3984
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s16 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s16 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s16 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s16 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3968
-; GFX6-NEXT: s_mov_b32 s12, 0x43d00
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3968
+; GFX6-NEXT: s_mov_b32 s4, 0x43d00
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x43900
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x43900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4080
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x43500
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x43500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4064
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x43100
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x43100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4048
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x42d00
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x42d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4032
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x42900
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x42900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4016
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4016
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x42500
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x42500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:4000
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s12, 0x42100
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x42100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3984
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s12 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s12 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s12 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s12 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x41d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3968
-; GFX6-NEXT: s_mov_b32 s8, 0x41d00
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[12:15], 0 addr64 offset:3968
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x41900
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x41900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4080
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4080
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x41500
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x41500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4064
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4064
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x41100
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x41100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4048
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4048
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x40d00
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x40900
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4032
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4032
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x40900
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x40500
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4016
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4016
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x40500
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x40100
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:4000
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:4000
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s8, 0x40100
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x3fd00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3984
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3984
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s8 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s8 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s8 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s4 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s4 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s4, 0x40d00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[4:7], 0 addr64 offset:3968
-; GFX6-NEXT: s_mov_b32 s4, 0x3fd00
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[7:8], s[8:11], 0 addr64 offset:3968
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s4 offset:4 ; 4-byte Folded Reload
@@ -4681,1296 +4679,1304 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
-; GFX9-FLATSCR-NEXT: s_mov_b32 s4, 4
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_mov_b32 s5, 4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0x84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x104
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x184
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x204
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x284
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x304
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x384
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x80
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_mov_b32 s4, 20
+; GFX9-FLATSCR-NEXT: s_mov_b32 s5, 20
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_mov_b32 s4, 36
+; GFX9-FLATSCR-NEXT: s_mov_b32 s5, 36
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_mov_b32 s4, 52
+; GFX9-FLATSCR-NEXT: s_mov_b32 s5, 52
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s5, 0x44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s5, 0x54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s5, 0x64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s5, 0x74
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s5 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s5, 0x100
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0x94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xa4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xc4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xd4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xe4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x180, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0xf4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s6 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x104
+; GFX9-FLATSCR-NEXT: s_movk_i32 s6, 0x180
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x114
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x114
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x124
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x124
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x134
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x134
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x144
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x144
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x154
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x154
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x164
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x164
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x200, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x174
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x174
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s7 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x184
+; GFX9-FLATSCR-NEXT: s_movk_i32 s7, 0x200
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x194
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x194
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x280, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x1f4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s8 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x204
+; GFX9-FLATSCR-NEXT: s_movk_i32 s8, 0x280
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x214
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x214
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x224
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x224
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x234
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x234
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x244
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x244
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x254
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x254
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x264
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x264
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x300, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x274
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x274
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s9 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x284
+; GFX9-FLATSCR-NEXT: s_movk_i32 s9, 0x300
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x294
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x294
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x380, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x2f4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s10 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x304
+; GFX9-FLATSCR-NEXT: s_movk_i32 s10, 0x380
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x314
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x314
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x324
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x324
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x334
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x334
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x344
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x344
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x354
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x354
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x364
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x364
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x400, v2
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x374
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x374
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s11 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x384
+; GFX9-FLATSCR-NEXT: s_movk_i32 s11, 0x400
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x394
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x394
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4064
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x3f4
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s12 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x3f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3]
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x404
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x404
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, s1
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:16
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x414
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x414
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:32
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x424
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x424
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:48
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x434
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x434
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:64
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x444
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x444
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:80
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x454
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x454
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:96
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x464
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x464
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:112
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x474
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x474
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:128
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x484
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x484
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:144
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x494
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x494
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:160
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:176
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:192
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:208
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:224
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:240
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x4f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x4f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:256
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x504
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x504
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:272
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x514
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x514
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:288
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x524
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x524
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:304
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x534
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x534
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:320
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x544
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x544
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:336
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x554
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x554
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:352
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x564
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x564
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:368
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x574
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x574
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:384
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x584
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x584
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:400
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x594
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x594
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:416
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:432
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:448
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:464
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:480
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:496
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x5f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x5f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:512
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x604
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x604
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:528
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x614
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x614
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:544
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x624
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x624
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:560
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x634
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x634
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:576
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x644
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x644
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:592
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x654
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x654
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:608
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x664
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x664
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:624
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x674
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x674
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:640
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x684
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x684
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:656
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x694
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x694
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:672
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:688
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:704
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:720
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:736
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:752
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x6f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x6f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:768
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x704
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x704
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:784
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x714
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x714
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:800
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x724
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x724
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:816
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x734
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x734
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:832
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x744
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x744
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:848
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x754
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x754
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:864
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x764
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x764
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:880
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x774
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x774
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:896
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x784
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x784
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:912
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x794
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x794
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:928
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:944
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:960
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:976
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:992
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1008
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x7f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x7f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1024
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x804
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x804
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1040
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x814
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x814
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1056
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x824
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x824
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1072
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x834
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x834
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1088
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x844
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x844
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1104
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x854
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x854
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1120
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x864
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x864
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1136
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x874
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x874
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1152
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x884
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x884
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1168
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x894
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x894
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1184
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1200
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1216
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1232
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1248
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1264
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x8f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x8f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1280
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x904
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x904
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1296
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x914
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x914
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1312
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x924
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x924
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1328
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x934
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x934
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1344
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x944
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x944
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1360
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x954
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x954
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1376
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x964
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x964
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1392
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x974
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x974
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1408
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x984
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x984
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1424
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x994
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x994
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1440
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1456
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1472
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1488
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1504
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1520
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x9f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x9f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1536
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1552
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1568
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1584
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1600
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1616
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1632
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1648
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1664
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1680
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xa94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xa94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1696
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xaa4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xaa4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1712
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xab4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xab4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1728
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xac4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xac4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1744
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xad4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xad4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1760
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xae4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xae4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1776
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xaf4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xaf4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1792
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1808
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1824
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1840
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1856
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1872
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1888
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1904
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1920
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1936
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xb94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xb94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1952
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xba4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xba4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xbb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xbb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:1984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xbc4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xbc4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xbd4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xbd4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xbe4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xbe4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xbf4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xbf4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2080
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2096
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2112
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2128
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2144
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2160
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2176
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2192
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xc94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xc94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2208
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xca4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xca4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2224
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xcb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xcb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2240
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xcc4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xcc4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2256
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xcd4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xcd4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2272
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xce4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xce4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2288
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xcf4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xcf4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2304
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2320
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2336
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2352
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2368
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2384
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2400
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2416
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2432
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2448
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xd94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xd94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2464
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xda4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xda4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2480
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xdb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xdb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2496
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xdc4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xdc4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2512
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xdd4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xdd4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2528
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xde4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xde4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2544
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xdf4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xdf4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2560
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2576
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2592
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2608
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2624
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2640
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2656
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2672
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2688
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2704
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xe94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xe94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2720
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xea4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xea4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2736
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xeb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xeb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2752
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xec4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xec4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2768
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xed4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xed4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2784
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xee4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xee4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2800
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xef4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xef4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2816
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf04
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf04
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2832
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf14
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf14
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2848
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf24
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf24
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2864
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf34
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf34
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2880
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf44
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf44
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2896
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf54
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf54
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2912
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf64
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2928
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf74
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2944
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf84
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2960
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf94
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xf94
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2976
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xfa4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xfa4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2992
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xfb4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xfb4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3008
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xfc4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xfc4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3024
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xfd4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xfd4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3040
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xfe4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xfe4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3056
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xff4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0xff4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3072
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1004
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1004
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3088
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1014
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1014
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3104
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1024
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1024
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3120
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1034
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1034
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3136
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1044
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1044
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3152
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1054
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1054
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3168
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1064
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1064
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3184
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1074
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1074
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3200
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1084
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1084
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3216
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1094
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1094
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3232
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3248
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3264
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3280
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3296
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3312
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x10f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x10f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3328
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1104
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1104
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3344
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1114
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1114
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3360
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1124
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1124
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3376
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1134
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1134
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3392
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1144
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1144
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3408
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1154
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1154
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3424
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1164
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1164
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3440
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1174
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1174
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3456
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1184
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1184
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3472
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1194
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1194
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3488
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3504
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3520
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3536
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3552
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3568
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x11f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x11f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3584
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1204
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1204
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3600
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1214
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1214
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3616
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1224
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1224
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3632
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1234
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1234
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3648
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1244
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1244
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3664
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1254
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1254
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3680
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1264
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1264
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3696
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1274
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1274
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3712
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1284
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1284
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3728
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1294
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1294
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3744
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3760
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3776
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3792
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3808
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3824
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x12f4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x12f4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3840
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1304
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1304
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3856
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1314
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1314
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3872
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1324
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1324
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3888
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1334
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1334
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3904
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1344
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1344
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3920
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1354
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1354
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3936
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1364
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1364
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3952
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1374
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1374
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3968
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1384
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1384
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:3984
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1394
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x1394
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4000
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x13a4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x13a4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4016
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x13b4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x13b4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4032
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x13c4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x13c4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4048
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x13d4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x13d4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4064
-; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x13e4
+; GFX9-FLATSCR-NEXT: s_movk_i32 s12, 0x13e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s12 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:4080
; GFX9-FLATSCR-NEXT: s_movk_i32 s2, 0x13e4
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
@@ -7012,7 +7018,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x3f4
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x400, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s11, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x3e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7046,7 +7052,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x380, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s10, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x364
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7080,7 +7086,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x300, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s9, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7114,7 +7120,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x280, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s8, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x264
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7148,7 +7154,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x200, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s7, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x1e4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7182,7 +7188,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x180, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s6, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x164
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7216,7 +7222,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s5, v4
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0xe4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -7249,7 +7255,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x74
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:3968
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x80, v4
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[2:5], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x64
@@ -8502,11 +8508,11 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s4 ; 16-byte Folded Spill
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] offset:2032
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v18, s2, s0, v5
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v19, null, s1, 0, s2
+; GFX10-FLATSCR-NEXT: s_movk_i32 s2, 0x13e4
; GFX10-FLATSCR-NEXT: ;;#ASMSTART
; GFX10-FLATSCR-NEXT: ;;#ASMEND
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v4, s2, s0, v5
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v6, null, s1, 0, s2
-; GFX10-FLATSCR-NEXT: s_movk_i32 s2, 0x13e4
; GFX10-FLATSCR-NEXT: ;;#ASMSTART
; GFX10-FLATSCR-NEXT: ;;#ASMEND
; GFX10-FLATSCR-NEXT: ;;#ASMSTART
@@ -9029,693 +9035,693 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xbf4
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v0, vcc_lo, 0x480, v4
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v0, vcc_lo, 0x480, v18
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: v_add_co_u32 v2, vcc_lo, 0x780, v0
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xbe4
; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xbd4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xbc4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xbb4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xba4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb94
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb84
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb74
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[7:10], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v2, vcc_lo, 0x400, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v2, vcc_lo, 0x400, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb64
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v7, vcc_lo, 0x780, v2
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v8, null, 0, v3, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v4, vcc_lo, 0x780, v2
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v3, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb54
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb44
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb34
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb24
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb14
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xb04
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[9:12], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xaf4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[9:12], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v7, vcc_lo, 0x380, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v8, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v4, vcc_lo, 0x380, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xae4
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v9, vcc_lo, 0x780, v7
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v8, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v6, vcc_lo, 0x780, v4
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v5, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xad4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xac4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xab4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xaa4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa94
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa84
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[8:11], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa74
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[11:14], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v9, vcc_lo, 0x300, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[8:11], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v6, vcc_lo, 0x300, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa64
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v11, vcc_lo, 0x780, v9
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v10, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v8, vcc_lo, 0x780, v6
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v7, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa54
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa44
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa34
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa24
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa14
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0xa04
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[13:16], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9f4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[13:16], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v11, vcc_lo, 0x280, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[10:13], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v8, vcc_lo, 0x280, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9e4
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v13, vcc_lo, 0x780, v11
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v12, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v10, vcc_lo, 0x780, v8
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v9, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9d4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9c4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9b4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x9a4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x994
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x984
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[12:15], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x974
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[15:18], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v13, vcc_lo, 0x200, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[12:15], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v10, vcc_lo, 0x200, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x964
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v15, vcc_lo, 0x780, v13
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v16, null, 0, v14, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v12, vcc_lo, 0x780, v10
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v13, null, 0, v11, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x954
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x944
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x934
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x924
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x914
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x904
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[17:20], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[14:17], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8f4
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[17:20], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v15, vcc_lo, 0x180, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v16, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8e4
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v17, vcc_lo, 0x780, v15
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v18, null, 0, v16, vcc_lo
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[14:17], off offset:1920
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v12, vcc_lo, 0x180, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v13, null, 0, v19, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v14, vcc_lo, 0x780, v12
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v15, null, 0, v13, vcc_lo
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8d4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8c4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8b4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x8a4
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x894
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x884
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x874
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[19:22], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v17, vcc_lo, 0x100, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v18, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[20:23], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v14, vcc_lo, 0x100, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v15, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x864
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v19, vcc_lo, 0x780, v17
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v20, null, 0, v18, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v16, vcc_lo, 0x780, v14
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v15, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x854
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x844
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x834
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x824
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x814
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x804
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, s0 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, s0 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[21:24], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:2036 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v19, vcc_lo, 0x80, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v20, null, 0, v6, vcc_lo
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x780, v19
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v20, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[20:23], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:2036 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v16, vcc_lo, 0x80, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v19, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x780, v16
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v17, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:2020 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:2020 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:2004 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:2004 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1988 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1988 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1972 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1972 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1956 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1956 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1940 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1940 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1924 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1924 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1908 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x780, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1908 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x780, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1892 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1892 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1876 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1876 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1860 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1860 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1844 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1844 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1828 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1828 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1812 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1812 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1796 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1796 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1780 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x700, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1780 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x700, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1764 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1764 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1748 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1748 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1732 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1732 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1716 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1716 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1700 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1700 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1684 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1684 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1668 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1668 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1652 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x680, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1652 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x680, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1636 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1636 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1620 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1620 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1604 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1604 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1588 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1588 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1572 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1572 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1556 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1556 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1540 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1540 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1524 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x600, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1524 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x600, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1508 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1508 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1492 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1492 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1476 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1476 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1460 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1460 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1444 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1444 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1428 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1428 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1412 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1412 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1396 ; 16-byte Folded Reload
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v21, vcc_lo, 0x580, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v6, vcc_lo
-; GFX10-FLATSCR-NEXT: v_add_co_u32 v4, vcc_lo, 0x500, v4
-; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v6, vcc_lo
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1396 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v20, vcc_lo, 0x580, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v19, vcc_lo
+; GFX10-FLATSCR-NEXT: v_add_co_u32 v18, vcc_lo, 0x500, v18
+; GFX10-FLATSCR-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1380 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1380 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1364 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1364 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1348 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1348 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1332 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1332 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1316 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1316 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1300 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1300 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[23:26], off, off offset:1284 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[22:25], off, off offset:1284 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[21:22], v[23:26], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1268 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[20:21], v[22:25], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1268 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1252 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1252 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1236 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1236 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1220 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1220 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1204 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1204 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1188 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1188 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1172 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1172 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1156 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[20:23], off, off offset:1156 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[21:24], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1140 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[18:19], v[20:23], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1140 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1124 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1124 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1108 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1108 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1092 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1092 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1076 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1076 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1060 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1060 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1044 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1044 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1028 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1028 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[21:24], off offset:1920
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:1012 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:1920
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:1012 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:2032
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:996 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:2032
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:996 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:2016
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:980 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:2016
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:980 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:2000
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:964 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:2000
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:964 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:1984
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:948 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:1984
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:948 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:1968
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:932 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:1968
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:932 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:1952
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:916 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:1952
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:916 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:1936
-; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[21:24], off, off offset:900 ; 16-byte Folded Reload
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:1936
+; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[18:21], off, off offset:900 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[21:24], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[2:3], v[18:21], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:884 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:868 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:852 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:836 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:820 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:804 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:788 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:772 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[4:5], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:756 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:740 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:724 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:708 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:692 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:676 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:660 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:644 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[9:10], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:628 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:612 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:596 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:580 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:564 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:548 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:532 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:516 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[11:12], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:500 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:484 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:468 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:452 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:436 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:420 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:404 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:388 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[13:14], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:372 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:356 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:340 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:324 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:308 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:292 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:276 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:260 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[15:16], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:244 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:228 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:212 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:196 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:180 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:164 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:148 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:132 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[17:18], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:116 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:2032
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:2032
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:100 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:2016
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:2016
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:84 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:2000
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:2000
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:68 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:1984
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:1984
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:52 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:1968
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:1968
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:36 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:1952
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:1952
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:20 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:1936
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:1936
; GFX10-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:4 ; 16-byte Folded Reload
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[19:20], v[0:3], off offset:1920
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:1920
; GFX10-FLATSCR-NEXT: s_endpgm
entry:
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
@@ -9751,7 +9757,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, -1, 0
; GFX6-NEXT: s_addc_u32 s41, s41, 0
; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, -1, v0
-; GFX6-NEXT: v_mov_b32_e32 v6, 0
+; GFX6-NEXT: v_mov_b32_e32 v10, 0
; GFX6-NEXT: s_mov_b64 s[4:5], exec
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
@@ -9769,9 +9775,9 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[4:5]
; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 8, v0
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:240
-; GFX6-NEXT: s_mov_b32 s2, 0x86800
+; GFX6-NEXT: v_lshlrev_b32_e32 v9, 8, v0
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:240
+; GFX6-NEXT: s_mov_b32 s2, 0x84c00
; GFX6-NEXT: s_mov_b64 s[8:9], exec
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -9779,137 +9785,155 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:224
-; GFX6-NEXT: s_mov_b32 s2, 0x86400
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:208
-; GFX6-NEXT: s_mov_b32 s2, 0x86000
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:192
-; GFX6-NEXT: s_mov_b32 s2, 0x85c00
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:176
-; GFX6-NEXT: s_mov_b32 s2, 0x85800
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:224
+; GFX6-NEXT: s_mov_b32 s2, 0x84800
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:160
-; GFX6-NEXT: s_mov_b32 s2, 0x85400
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:208
+; GFX6-NEXT: s_mov_b32 s2, 0x84400
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_load_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64 offset:192
+; GFX6-NEXT: buffer_load_dwordx4 v[19:22], v[9:10], s[4:7], 0 addr64 offset:176
+; GFX6-NEXT: buffer_load_dwordx4 v[11:14], v[9:10], s[4:7], 0 addr64 offset:160
+; GFX6-NEXT: buffer_load_dwordx4 v[4:7], v[9:10], s[4:7], 0 addr64 offset:144
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:144
-; GFX6-NEXT: s_mov_b32 s2, 0x85000
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:128
+; GFX6-NEXT: s_mov_b32 s2, 0x83000
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:128
+; GFX6-NEXT: s_mov_b32 s2, 0x83400
+; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v5, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v6, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s2, 0x83800
+; GFX6-NEXT: buffer_store_dword v11, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v12, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v13, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v14, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s2, 0x83c00
+; GFX6-NEXT: buffer_store_dword v19, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v20, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v21, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v22, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s2, 0x84000
+; GFX6-NEXT: buffer_store_dword v15, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v16, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v17, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v18, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_mov_b32 s2, 0x84c00
-; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s2 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s2, 0x85000
+; GFX6-NEXT: s_waitcnt vmcnt(3)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
+; GFX6-NEXT: s_waitcnt vmcnt(3)
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: s_waitcnt vmcnt(3)
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: s_waitcnt vmcnt(3)
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:112
-; GFX6-NEXT: s_mov_b32 s2, 0x84800
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:112
+; GFX6-NEXT: s_mov_b32 s2, 0x86c00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:96
-; GFX6-NEXT: s_mov_b32 s2, 0x84400
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:96
+; GFX6-NEXT: s_mov_b32 s2, 0x86800
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:80
-; GFX6-NEXT: s_mov_b32 s2, 0x84000
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:80
+; GFX6-NEXT: s_mov_b32 s2, 0x86400
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:64
-; GFX6-NEXT: s_mov_b32 s2, 0x83800
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:64
+; GFX6-NEXT: s_mov_b32 s2, 0x86000
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_load_dwordx4 v[15:18], v[9:10], s[4:7], 0 addr64
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64
-; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:16
-; GFX6-NEXT: s_mov_b32 s2, 0x83000
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v9, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v10, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:32
-; GFX6-NEXT: s_mov_b32 s2, 0x83400
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s2 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v9, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v10, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[9:10], s[4:7], 0 addr64 offset:16
+; GFX6-NEXT: buffer_load_dwordx4 v[4:7], v[9:10], s[4:7], 0 addr64 offset:32
; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7]
; GFX6-NEXT: s_mov_b64 exec, 15
-; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
+; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_writelane_b32 v4, s0, 0
-; GFX6-NEXT: v_writelane_b32 v4, s1, 1
-; GFX6-NEXT: v_writelane_b32 v4, s2, 2
-; GFX6-NEXT: v_writelane_b32 v4, s3, 3
+; GFX6-NEXT: v_writelane_b32 v8, s0, 0
+; GFX6-NEXT: v_writelane_b32 v8, s1, 1
+; GFX6-NEXT: v_writelane_b32 v8, s2, 2
+; GFX6-NEXT: v_writelane_b32 v8, s3, 3
; GFX6-NEXT: s_mov_b32 s10, 0x80800
-; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], s10 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], s10 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
+; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[8:9]
-; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:48
-; GFX6-NEXT: s_mov_b32 s0, 0x83c00
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 13, v0
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 16, v4
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s0 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Spill
-; GFX6-NEXT: buffer_store_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_load_dwordx4 v[11:14], v[9:10], s[4:7], 0 addr64 offset:48
+; GFX6-NEXT: v_mov_b32_e32 v19, v15
+; GFX6-NEXT: v_mov_b32_e32 v20, v16
+; GFX6-NEXT: v_mov_b32_e32 v21, v17
+; GFX6-NEXT: v_mov_b32_e32 v22, v18
+; GFX6-NEXT: s_mov_b32 s0, 0x84c00
+; GFX6-NEXT: buffer_store_dword v19, off, s[40:43], s0 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v20, off, s[40:43], s0 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v21, off, s[40:43], s0 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v22, off, s[40:43], s0 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s0, 0x85400
+; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s0 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s0, 0x85800
+; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], s0 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s0, 0x85c00
; GFX6-NEXT: s_waitcnt expcnt(3)
-; GFX6-NEXT: v_mov_b32_e32 v7, 1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 13, v15
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 16, v4
+; GFX6-NEXT: s_waitcnt expcnt(2)
+; GFX6-NEXT: v_mov_b32_e32 v5, 1
+; GFX6-NEXT: buffer_store_dword v5, v4, s[40:43], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(13)
+; GFX6-NEXT: buffer_store_dword v11, off, s[40:43], s0 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v12, off, s[40:43], s0 offset:4 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v13, off, s[40:43], s0 offset:8 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_store_dword v14, off, s[40:43], s0 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: s_mov_b32 s0, 0x86c00
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b64 s[0:1], exec
-; GFX6-NEXT: buffer_store_dword v7, v4, s[40:43], 0 offen
; GFX6-NEXT: ;;#ASMSTART
; GFX6-NEXT: ; def s[4:11]
; GFX6-NEXT: ;;#ASMEND
@@ -9930,7 +9954,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[0:1]
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v15
; GFX6-NEXT: ;;#ASMSTART
; GFX6-NEXT: ; def s[8:15]
; GFX6-NEXT: ;;#ASMEND
@@ -10096,9 +10120,9 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: s_mov_b64 s[44:45], exec
; GFX6-NEXT: s_mov_b64 exec, 15
; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x20b0
+; GFX6-NEXT: v_mov_b32_e32 v5, 0x20b0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, v7, s[40:43], 0 offen ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v4, v5, s[40:43], 0 offen ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readlane_b32 s36, v4, 0
; GFX6-NEXT: v_readlane_b32 s37, v4, 1
@@ -10155,127 +10179,127 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[4:5]
-; GFX6-NEXT: s_mov_b32 s0, 0x86800
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x85000
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b64 s[38:39], s[2:3]
-; GFX6-NEXT: s_mov_b32 s0, 0x86400
+; GFX6-NEXT: s_mov_b32 s0, 0x84800
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:240
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x86000
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x84400
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:224
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:224
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85c00
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x84000
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:208
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:208
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85800
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x83c00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:192
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:192
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85400
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x83800
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:176
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:176
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85000
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x83400
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:160
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:160
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84c00
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x83000
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:144
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:144
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84800
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v5, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v6, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x86800
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:128
+; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[9:10], s[36:39], 0 addr64 offset:128
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:112
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84400
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x86400
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:112
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:96
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84000
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x86000
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:96
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:80
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83800
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x85c00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:80
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:64
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83c00
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x85800
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:64
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:48
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83400
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x85400
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:48
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:32
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83000
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: s_mov_b32 s0, 0x84c00
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:32
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64 offset:16
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
-; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v0, off, s[40:43], s0 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v1, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v2, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v3, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:16
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[5:6], s[36:39], 0 addr64
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], v[9:10], s[36:39], 0 addr64
; GFX6-NEXT: s_endpgm
;
; GFX9-FLATSCR-LABEL: test_limited_sgpr:
@@ -10283,66 +10307,90 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0
-; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v5, 8, v0
+; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v9, 8, v0
; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s8, s13
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:240
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:240
; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 16
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:144
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:224
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:224
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:208
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:208
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[14:17], v9, s[38:39] offset:192
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[18:21], v9, s[38:39] offset:176
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[10:13], v9, s[38:39] offset:160
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[4:7], v9, s[38:39] offset:144
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:128
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:192
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:176
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:160
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[4:7], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[14:17], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:128
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:112
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:112
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20d0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:96
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:96
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:80
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:32
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:16
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(2)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:80
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:64
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:48
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[10:13], v9, s[38:39] offset:48
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[4:7], v9, s[38:39] offset:32
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v9, s[38:39] offset:16
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[14:17], v9, s[38:39]
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(2)
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v22, v7
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[7:10], v5, s[38:39]
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 1
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: v_lshl_add_u32 v4, v7, 13, v4
-; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX9-FLATSCR-NEXT: scratch_store_dword v4, v6, off
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v14
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, v15
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v16
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v21, v17
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[18:21], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20d0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v15, v0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v16, v1
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v17, v2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v18, v3
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, v4
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 16
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v5
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v21, v6
+; GFX9-FLATSCR-NEXT: v_lshl_add_u32 v4, v14, 13, v4
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v5, 1
+; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v14
+; GFX9-FLATSCR-NEXT: scratch_store_dword v4, v5, off
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ; def s[0:7]
; GFX9-FLATSCR-NEXT: ;;#ASMEND
@@ -10375,22 +10423,16 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[19:22], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20f0
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, v11
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v12
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v13
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v14
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(4)
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[10:13], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_nop 0
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ;;#ASMEND
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[10:13], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20d0
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v14, v3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, v2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, v1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, v0
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ;;#ASMEND
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
@@ -10405,58 +10447,58 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: ;;#ASMEND
; GFX9-FLATSCR-NEXT: .LBB1_2: ; %ret
; GFX9-FLATSCR-NEXT: s_or_b64 exec, exec, s[34:35]
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:112
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:96
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:80
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:64
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:240
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:48
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:32
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:224
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:16
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37]
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:208
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:240
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:192
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:224
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:176
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:160
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:144
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[4:7], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[4:7], s[36:37] offset:128
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[0:3], s[36:37] offset:112
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:208
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:192
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:176
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(3)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:160
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[0:3], s[36:37] offset:96
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[0:3], s[36:37] offset:80
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[10:13], s[36:37] offset:64
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:144
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[0:3], s[36:37] offset:48
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[19:22], s[36:37] offset:32
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[15:18], s[36:37] offset:16
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:128
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v9, v[0:3], s[36:37]
; GFX9-FLATSCR-NEXT: s_endpgm
;
; GFX10-FLATSCR-LABEL: test_limited_sgpr:
@@ -10650,6 +10692,14 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: ;;#ASMEND
; GFX10-FLATSCR-NEXT: .LBB1_2: ; %ret
; GFX10-FLATSCR-NEXT: s_or_b32 exec_lo, exec_lo, s33
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[35:38], s[36:37] offset:240
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[31:34], s[36:37] offset:224
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[27:30], s[36:37] offset:208
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[23:26], s[36:37] offset:192
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:176
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:160
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:144
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37] offset:128
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[63:66], s[36:37] offset:112
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[59:62], s[36:37] offset:96
@@ -10659,14 +10709,6 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[43:46], s[36:37] offset:32
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[39:42], s[36:37] offset:16
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37]
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[35:38], s[36:37] offset:240
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[31:34], s[36:37] offset:224
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[27:30], s[36:37] offset:208
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[23:26], s[36:37] offset:192
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:176
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:160
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:144
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37] offset:128
; GFX10-FLATSCR-NEXT: s_endpgm
entry:
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 05dba10e9d9ab..64e673af67e3a 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -137,7 +137,7 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
; GFX900: NumVgprs: 256
; GFX908: NumVgprs: 254
-; GFX900: ScratchSize: 132
+; GFX900: ScratchSize: 148
; GFX908: ScratchSize: 0
; GFX900: VGPRBlocks: 63
; GFX908: VGPRBlocks: 63
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index d4a4f6ed9eefa..c10a67205fd2e 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -31,8 +31,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr10
; CHECK-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr8
; CHECK-NEXT: undef [[S_LOAD_DWORDX2_IMM:%[0-9]+]].sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 232, 0 :: (invariant load (s64) from %ir.39, addrspace 4)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %124:sgpr_128, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
- ; CHECK-NEXT: KILL undef %124:sgpr_128
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %121:sgpr_128, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL undef %121:sgpr_128
; CHECK-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], 4, implicit-def dead $scc
; CHECK-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], 4, implicit-def dead $scc
; CHECK-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], 4, implicit-def dead $scc
@@ -51,13 +51,13 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_LOAD_DWORDX4_IMM]], 0, 0 :: (dereferenceable invariant load (s32) from %ir.84, align 1, addrspace 8)
; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %117:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %114:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
- ; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %88:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %85:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], %67.sub0_sub1_sub2_sub3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL undef %88:sgpr_128
+ ; CHECK-NEXT: KILL undef %85:sgpr_128
; CHECK-NEXT: [[S_SUB_I32_2:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM1]], 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_1:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
@@ -67,8 +67,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.89, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.95, addrspace 4)
; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %170:sreg_32, 31, implicit-def dead $scc
- ; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %170:sreg_32, implicit-def $scc
+ ; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %167:sreg_32, 31, implicit-def dead $scc
+ ; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %167:sreg_32, implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_4:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_5:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.101, addrspace 4)
@@ -77,7 +77,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_ADD_U32_6:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_1]], implicit-def $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 0, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
; CHECK-NEXT: [[S_ADD_U32_6:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_1]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, undef %170:sreg_32, implicit-def $scc
+ ; CHECK-NEXT: undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, undef %167:sreg_32, implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_7:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_8:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_8:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
@@ -90,18 +90,18 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_]], 16, implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_2]], 16, implicit-def dead $scc
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) null`, align 1, addrspace 8)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], undef %297:sreg_32, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], undef %294:sreg_32, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) null`, align 1, addrspace 8)
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) null`, align 1, addrspace 8)
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_MOV_B32_]], 16, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) null`, align 1, addrspace 8)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %126:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %352:sgpr_128, undef %353:sreg_32, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %123:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %349:sgpr_128, undef %350:sreg_32, 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM6:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 224, 0 :: (invariant load (s128) from %ir.131, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM7:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 0, 0 :: (invariant load (s128) from %ir.119, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM8:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 224, 0 :: (invariant load (s128) from %ir.136, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM9:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 0, 0 :: (invariant load (s128) from %ir.126, addrspace 4)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %347:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %358:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %344:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %355:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM2]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN4:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM3]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN5:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM4]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
@@ -119,7 +119,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_ADD_U32_15:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY11]], [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_15:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %39:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_LSHL4_ADD_U32_:%[0-9]+]]:sreg_32 = S_LSHL4_ADD_U32 [[COPY12]], 16, implicit-def dead $scc
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %376:sgpr_128, [[S_LSHL4_ADD_U32_]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %373:sgpr_128, [[S_LSHL4_ADD_U32_]], 0, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(8) poison`, align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN7:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM7]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM10:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_8]], 224, 0 :: (invariant load (s128) from %ir.142, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM11:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY7]], 224, 0 :: (invariant load (s128) from %ir.147, addrspace 4)
@@ -228,10 +228,10 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[V_MOV_B32_e32_]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM26]]
; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM24]]
; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM25]]
+ ; CHECK-NEXT: KILL [[V_MOV_B32_e32_]]
+ ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM26]]
; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -2, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -1, [[BUFFER_LOAD_FORMAT_X_IDXEN1]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -3, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
@@ -348,7 +348,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[V_OR_B32_e64_67:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[S_ADD_I32_23]], [[V_OR_B32_e64_66]], implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 0, [[V_OR_B32_e64_67]], implicit $exec
; CHECK-NEXT: undef [[V_CNDMASK_B32_e64_:%[0-9]+]].sub3:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
- ; CHECK-NEXT: IMAGE_STORE_V4_V2_nsa_gfx10 [[V_CNDMASK_B32_e64_]], undef %541:vgpr_32, undef %543:vgpr_32, %67, 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
+ ; CHECK-NEXT: IMAGE_STORE_V4_V2_nsa_gfx10 [[V_CNDMASK_B32_e64_]], undef %538:vgpr_32, undef %540:vgpr_32, %67, 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
; CHECK-NEXT: S_ENDPGM 0
.expVert:
%0 = extractelement <31 x i32> %userData, i64 2
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index 66807cc42c12f..063af59339ba3 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -293,21 +293,21 @@ define amdgpu_kernel void @ashr_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_mov_b32 s0, s4
; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: v_readfirstlane_b32 s5, v2
-; VI-NEXT: v_readfirstlane_b32 s6, v1
-; VI-NEXT: v_readfirstlane_b32 s7, v3
-; VI-NEXT: s_lshr_b32 s8, s7, 16
-; VI-NEXT: s_ashr_i32 s9, s6, 16
-; VI-NEXT: s_sext_i32_i16 s6, s6
-; VI-NEXT: s_lshr_b32 s10, s5, 16
-; VI-NEXT: s_ashr_i32 s11, s4, 16
-; VI-NEXT: s_sext_i32_i16 s4, s4
-; VI-NEXT: s_ashr_i32 s8, s9, s8
-; VI-NEXT: s_ashr_i32 s6, s6, s7
-; VI-NEXT: s_ashr_i32 s7, s11, s10
-; VI-NEXT: s_ashr_i32 s4, s4, s5
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: v_readfirstlane_b32 s4, v2
+; VI-NEXT: v_readfirstlane_b32 s5, v0
+; VI-NEXT: v_readfirstlane_b32 s6, v3
+; VI-NEXT: v_readfirstlane_b32 s7, v1
+; VI-NEXT: s_lshr_b32 s8, s4, 16
+; VI-NEXT: s_lshr_b32 s9, s6, 16
+; VI-NEXT: s_ashr_i32 s10, s7, 16
+; VI-NEXT: s_sext_i32_i16 s7, s7
+; VI-NEXT: s_ashr_i32 s11, s5, 16
+; VI-NEXT: s_sext_i32_i16 s5, s5
+; VI-NEXT: s_ashr_i32 s9, s10, s9
+; VI-NEXT: s_ashr_i32 s6, s7, s6
+; VI-NEXT: s_ashr_i32 s7, s11, s8
+; VI-NEXT: s_ashr_i32 s4, s5, s4
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_and_b32 s6, s6, 0xffff
; VI-NEXT: s_lshl_b32 s7, s7, 16
; VI-NEXT: s_and_b32 s4, s4, 0xffff
diff --git a/llvm/test/CodeGen/AMDGPU/srem.ll b/llvm/test/CodeGen/AMDGPU/srem.ll
index 56b232d90b4c0..d37a8ac30ce4c 100644
--- a/llvm/test/CodeGen/AMDGPU/srem.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem.ll
@@ -2717,11 +2717,11 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_readfirstlane_b32 s11, v5
; GCN-NEXT: v_readfirstlane_b32 s10, v4
; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[8:9]
-; GCN-NEXT: v_readfirstlane_b32 s3, v3
-; GCN-NEXT: v_readfirstlane_b32 s2, v2
+; GCN-NEXT: v_readfirstlane_b32 s4, v6
; GCN-NEXT: v_readfirstlane_b32 s5, v7
+; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: s_cmp_lg_u32 s7, 0
-; GCN-NEXT: v_readfirstlane_b32 s4, v6
+; GCN-NEXT: v_readfirstlane_b32 s3, v3
; GCN-NEXT: s_cbranch_scc0 .LBB10_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_ashr_i32 s6, s9, 31
@@ -3016,30 +3016,29 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: ; %bb.9:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2
; GCN-NEXT: s_sub_i32 s3, 0, s2
-; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: s_mov_b32 s11, 0
; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT: v_mul_lo_u32 v1, s3, v0
-; GCN-NEXT: v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT: v_add_u32_e32 v0, v0, v1
-; GCN-NEXT: v_mul_hi_u32 v0, s4, v0
-; GCN-NEXT: v_mul_lo_u32 v0, v0, s2
-; GCN-NEXT: v_sub_u32_e32 v0, s4, v0
-; GCN-NEXT: v_subrev_u32_e32 v1, s2, v0
-; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT: v_subrev_u32_e32 v1, s2, v0
-; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0
-; GCN-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-NEXT: s_branch .LBB10_11
-; GCN-NEXT: .LBB10_10:
-; GCN-NEXT: v_mov_b32_e32 v2, s10
-; GCN-NEXT: v_mov_b32_e32 v3, s11
-; GCN-NEXT: .LBB10_11: ; %.split.split
+; GCN-NEXT: v_readfirstlane_b32 s5, v0
+; GCN-NEXT: s_mul_i32 s3, s3, s5
+; GCN-NEXT: s_mul_hi_u32 s3, s5, s3
+; GCN-NEXT: s_add_i32 s5, s5, s3
+; GCN-NEXT: s_mul_hi_u32 s3, s4, s5
+; GCN-NEXT: s_mul_i32 s3, s3, s2
+; GCN-NEXT: s_sub_i32 s3, s4, s3
+; GCN-NEXT: s_sub_i32 s4, s3, s2
+; GCN-NEXT: s_cmp_ge_u32 s3, s2
+; GCN-NEXT: s_cselect_b32 s3, s4, s3
+; GCN-NEXT: s_sub_i32 s4, s3, s2
+; GCN-NEXT: s_cmp_ge_u32 s3, s2
+; GCN-NEXT: s_cselect_b32 s10, s4, s3
+; GCN-NEXT: .LBB10_10: ; %.split.split
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: v_mov_b32_e32 v1, s7
+; GCN-NEXT: v_mov_b32_e32 v2, s10
+; GCN-NEXT: v_mov_b32_e32 v3, s11
; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GCN-NEXT: s_endpgm
;
@@ -3051,164 +3050,18 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TAHITI-NEXT: s_waitcnt lgkmcnt(0)
; TAHITI-NEXT: s_mov_b32 s0, s6
; TAHITI-NEXT: s_mov_b32 s1, s7
-; TAHITI-NEXT: buffer_load_dwordx4 v[4:7], off, s[0:3], 0
-; TAHITI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; TAHITI-NEXT: buffer_load_dwordx4 v[6:9], off, s[0:3], 0
+; TAHITI-NEXT: buffer_load_dwordx4 v[2:5], off, s[0:3], 0 offset:16
+; TAHITI-NEXT: s_waitcnt vmcnt(1)
+; TAHITI-NEXT: v_readfirstlane_b32 s8, v8
; TAHITI-NEXT: s_waitcnt vmcnt(0)
-; TAHITI-NEXT: v_or_b32_e32 v8, v5, v1
-; TAHITI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; TAHITI-NEXT: s_cbranch_vccz .LBB10_2
-; TAHITI-NEXT: ; %bb.1:
-; TAHITI-NEXT: v_ashrrev_i32_e32 v9, 31, v1
-; TAHITI-NEXT: v_add_i32_e32 v8, vcc, v0, v9
-; TAHITI-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
-; TAHITI-NEXT: v_xor_b32_e32 v8, v8, v9
-; TAHITI-NEXT: v_xor_b32_e32 v1, v1, v9
-; TAHITI-NEXT: v_cvt_f32_u32_e32 v9, v8
-; TAHITI-NEXT: v_cvt_f32_u32_e32 v10, v1
-; TAHITI-NEXT: v_sub_i32_e32 v11, vcc, 0, v8
-; TAHITI-NEXT: v_subb_u32_e32 v12, vcc, 0, v1, vcc
-; TAHITI-NEXT: v_madmk_f32 v9, v10, 0x4f800000, v9
-; TAHITI-NEXT: v_rcp_f32_e32 v9, v9
-; TAHITI-NEXT: s_mov_b64 s[6:7], 0
-; TAHITI-NEXT: v_mul_f32_e32 v9, 0x5f7ffffc, v9
-; TAHITI-NEXT: v_mul_f32_e32 v10, 0x2f800000, v9
-; TAHITI-NEXT: v_trunc_f32_e32 v10, v10
-; TAHITI-NEXT: v_madmk_f32 v9, v10, 0xcf800000, v9
-; TAHITI-NEXT: v_cvt_u32_f32_e32 v10, v10
-; TAHITI-NEXT: v_cvt_u32_f32_e32 v9, v9
-; TAHITI-NEXT: v_mul_lo_u32 v14, v11, v10
-; TAHITI-NEXT: v_mul_hi_u32 v13, v11, v9
-; TAHITI-NEXT: v_mul_lo_u32 v15, v12, v9
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; TAHITI-NEXT: v_mul_lo_u32 v14, v11, v9
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; TAHITI-NEXT: v_mul_lo_u32 v15, v9, v13
-; TAHITI-NEXT: v_mul_hi_u32 v16, v9, v14
-; TAHITI-NEXT: v_mul_hi_u32 v17, v9, v13
-; TAHITI-NEXT: v_mul_hi_u32 v18, v10, v13
-; TAHITI-NEXT: v_mul_lo_u32 v13, v10, v13
-; TAHITI-NEXT: v_add_i32_e32 v15, vcc, v16, v15
-; TAHITI-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v17, v10, v14
-; TAHITI-NEXT: v_mul_hi_u32 v14, v10, v14
-; TAHITI-NEXT: v_add_i32_e32 v15, vcc, v15, v17
-; TAHITI-NEXT: v_addc_u32_e32 v14, vcc, v16, v14, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v15, vcc, 0, v18, vcc
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; TAHITI-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
-; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, v10, v14, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v13, v11, v9
-; TAHITI-NEXT: v_mul_lo_u32 v14, v11, v10
-; TAHITI-NEXT: v_mul_hi_u32 v11, v11, v9
-; TAHITI-NEXT: v_mul_lo_u32 v12, v12, v9
-; TAHITI-NEXT: v_mul_hi_u32 v15, v10, v13
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v14, v11
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; TAHITI-NEXT: v_mul_lo_u32 v12, v9, v11
-; TAHITI-NEXT: v_mul_hi_u32 v14, v9, v13
-; TAHITI-NEXT: v_mul_hi_u32 v16, v9, v11
-; TAHITI-NEXT: v_mul_lo_u32 v13, v10, v13
-; TAHITI-NEXT: v_add_i32_e32 v12, vcc, v14, v12
-; TAHITI-NEXT: v_addc_u32_e32 v14, vcc, 0, v16, vcc
-; TAHITI-NEXT: v_mul_hi_u32 v16, v10, v11
-; TAHITI-NEXT: v_mul_lo_u32 v11, v10, v11
-; TAHITI-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, v14, v15, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v13, vcc, 0, v16, vcc
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
-; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v9, v11
-; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, v10, v12, vcc
-; TAHITI-NEXT: v_ashrrev_i32_e32 v11, 31, v5
-; TAHITI-NEXT: v_add_i32_e32 v12, vcc, v4, v11
-; TAHITI-NEXT: v_xor_b32_e32 v12, v12, v11
-; TAHITI-NEXT: v_mul_lo_u32 v13, v12, v10
-; TAHITI-NEXT: v_mul_hi_u32 v14, v12, v9
-; TAHITI-NEXT: v_mul_hi_u32 v15, v12, v10
-; TAHITI-NEXT: v_addc_u32_e32 v5, vcc, v5, v11, vcc
-; TAHITI-NEXT: v_xor_b32_e32 v5, v5, v11
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; TAHITI-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v15, v5, v9
-; TAHITI-NEXT: v_mul_hi_u32 v9, v5, v9
-; TAHITI-NEXT: v_mul_hi_u32 v16, v5, v10
-; TAHITI-NEXT: v_mul_lo_u32 v10, v5, v10
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; TAHITI-NEXT: v_addc_u32_e32 v9, vcc, v14, v9, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v13, vcc, 0, v16, vcc
-; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v10, v8, v10
-; TAHITI-NEXT: v_mul_hi_u32 v13, v8, v9
-; TAHITI-NEXT: v_mul_lo_u32 v14, v1, v9
-; TAHITI-NEXT: v_mul_lo_u32 v9, v8, v9
-; TAHITI-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; TAHITI-NEXT: v_add_i32_e32 v10, vcc, v14, v10
-; TAHITI-NEXT: v_sub_i32_e32 v13, vcc, v5, v10
-; TAHITI-NEXT: v_sub_i32_e32 v9, vcc, v12, v9
-; TAHITI-NEXT: v_subb_u32_e64 v12, s[0:1], v13, v1, vcc
-; TAHITI-NEXT: v_sub_i32_e64 v13, s[0:1], v9, v8
-; TAHITI-NEXT: v_subbrev_u32_e64 v14, s[2:3], 0, v12, s[0:1]
-; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v14, v1
-; TAHITI-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[2:3]
-; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v13, v8
-; TAHITI-NEXT: v_subb_u32_e32 v5, vcc, v5, v10, vcc
-; TAHITI-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[2:3]
-; TAHITI-NEXT: v_cmp_eq_u32_e64 s[2:3], v14, v1
-; TAHITI-NEXT: v_subb_u32_e64 v12, s[0:1], v12, v1, s[0:1]
-; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v5, v1
-; TAHITI-NEXT: v_cndmask_b32_e64 v15, v15, v16, s[2:3]
-; TAHITI-NEXT: v_sub_i32_e64 v16, s[0:1], v13, v8
-; TAHITI-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v9, v8
-; TAHITI-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v12, s[0:1]
-; TAHITI-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; TAHITI-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
-; TAHITI-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v15
-; TAHITI-NEXT: v_cndmask_b32_e32 v1, v10, v8, vcc
-; TAHITI-NEXT: v_cndmask_b32_e64 v13, v13, v16, s[0:1]
-; TAHITI-NEXT: v_cndmask_b32_e64 v12, v14, v12, s[0:1]
-; TAHITI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; TAHITI-NEXT: v_cndmask_b32_e32 v1, v5, v12, vcc
-; TAHITI-NEXT: v_cndmask_b32_e32 v5, v9, v13, vcc
-; TAHITI-NEXT: v_xor_b32_e32 v5, v5, v11
-; TAHITI-NEXT: v_xor_b32_e32 v1, v1, v11
-; TAHITI-NEXT: v_sub_i32_e32 v8, vcc, v5, v11
-; TAHITI-NEXT: v_subb_u32_e32 v9, vcc, v1, v11, vcc
-; TAHITI-NEXT: s_branch .LBB10_3
-; TAHITI-NEXT: .LBB10_2:
-; TAHITI-NEXT: s_mov_b64 s[6:7], -1
-; TAHITI-NEXT: ; implicit-def: $vgpr8_vgpr9
-; TAHITI-NEXT: .LBB10_3: ; %Flow5
-; TAHITI-NEXT: s_and_b64 s[0:1], s[6:7], exec
-; TAHITI-NEXT: s_cselect_b32 s0, 1, 0
-; TAHITI-NEXT: s_cmp_lg_u32 s0, 1
-; TAHITI-NEXT: s_cbranch_scc1 .LBB10_5
-; TAHITI-NEXT: ; %bb.4:
-; TAHITI-NEXT: v_cvt_f32_u32_e32 v1, v0
-; TAHITI-NEXT: v_sub_i32_e32 v5, vcc, 0, v0
-; TAHITI-NEXT: v_mov_b32_e32 v9, 0
-; TAHITI-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; TAHITI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; TAHITI-NEXT: v_cvt_u32_f32_e32 v1, v1
-; TAHITI-NEXT: v_mul_lo_u32 v5, v5, v1
-; TAHITI-NEXT: v_mul_hi_u32 v5, v1, v5
-; TAHITI-NEXT: v_add_i32_e32 v1, vcc, v1, v5
-; TAHITI-NEXT: v_mul_hi_u32 v1, v4, v1
-; TAHITI-NEXT: v_mul_lo_u32 v1, v1, v0
-; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, v4, v1
-; TAHITI-NEXT: v_sub_i32_e32 v4, vcc, v1, v0
-; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0
-; TAHITI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; TAHITI-NEXT: v_sub_i32_e32 v4, vcc, v1, v0
-; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0
-; TAHITI-NEXT: v_cndmask_b32_e32 v8, v1, v4, vcc
-; TAHITI-NEXT: .LBB10_5: ; %.split
; TAHITI-NEXT: v_or_b32_e32 v0, v7, v3
; TAHITI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; TAHITI-NEXT: s_cbranch_vccz .LBB10_7
-; TAHITI-NEXT: ; %bb.6:
+; TAHITI-NEXT: v_readfirstlane_b32 s9, v9
+; TAHITI-NEXT: v_readfirstlane_b32 s6, v4
+; TAHITI-NEXT: v_readfirstlane_b32 s7, v5
+; TAHITI-NEXT: s_cbranch_vccz .LBB10_2
+; TAHITI-NEXT: ; %bb.1:
; TAHITI-NEXT: v_ashrrev_i32_e32 v0, 31, v3
; TAHITI-NEXT: v_add_i32_e32 v1, vcc, v2, v0
; TAHITI-NEXT: v_addc_u32_e32 v3, vcc, v3, v0, vcc
@@ -3217,128 +3070,128 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TAHITI-NEXT: v_cvt_f32_u32_e32 v3, v1
; TAHITI-NEXT: v_cvt_f32_u32_e32 v4, v0
; TAHITI-NEXT: v_sub_i32_e32 v5, vcc, 0, v1
-; TAHITI-NEXT: v_subb_u32_e32 v10, vcc, 0, v0, vcc
+; TAHITI-NEXT: v_subb_u32_e32 v8, vcc, 0, v0, vcc
; TAHITI-NEXT: v_madmk_f32 v3, v4, 0x4f800000, v3
; TAHITI-NEXT: v_rcp_f32_e32 v3, v3
-; TAHITI-NEXT: s_mov_b64 s[6:7], 0
+; TAHITI-NEXT: s_mov_b64 s[10:11], 0
; TAHITI-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; TAHITI-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; TAHITI-NEXT: v_trunc_f32_e32 v4, v4
; TAHITI-NEXT: v_madmk_f32 v3, v4, 0xcf800000, v3
; TAHITI-NEXT: v_cvt_u32_f32_e32 v4, v4
; TAHITI-NEXT: v_cvt_u32_f32_e32 v3, v3
-; TAHITI-NEXT: v_mul_lo_u32 v12, v5, v4
-; TAHITI-NEXT: v_mul_hi_u32 v11, v5, v3
-; TAHITI-NEXT: v_mul_lo_u32 v13, v10, v3
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; TAHITI-NEXT: v_mul_lo_u32 v12, v5, v3
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; TAHITI-NEXT: v_mul_lo_u32 v13, v3, v11
-; TAHITI-NEXT: v_mul_hi_u32 v14, v3, v12
-; TAHITI-NEXT: v_mul_hi_u32 v15, v3, v11
-; TAHITI-NEXT: v_mul_hi_u32 v16, v4, v11
-; TAHITI-NEXT: v_mul_lo_u32 v11, v4, v11
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; TAHITI-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v15, v4, v12
-; TAHITI-NEXT: v_mul_hi_u32 v12, v4, v12
-; TAHITI-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, v14, v12, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v13, vcc, 0, v16, vcc
+; TAHITI-NEXT: v_mul_lo_u32 v10, v5, v4
+; TAHITI-NEXT: v_mul_hi_u32 v9, v5, v3
+; TAHITI-NEXT: v_mul_lo_u32 v11, v8, v3
+; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; TAHITI-NEXT: v_mul_lo_u32 v10, v5, v3
+; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; TAHITI-NEXT: v_mul_lo_u32 v11, v3, v9
+; TAHITI-NEXT: v_mul_hi_u32 v12, v3, v10
+; TAHITI-NEXT: v_mul_hi_u32 v13, v3, v9
+; TAHITI-NEXT: v_mul_hi_u32 v14, v4, v9
+; TAHITI-NEXT: v_mul_lo_u32 v9, v4, v9
; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
-; TAHITI-NEXT: v_add_i32_e32 v3, vcc, v3, v11
-; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, v4, v12, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v11, v5, v3
-; TAHITI-NEXT: v_mul_lo_u32 v12, v5, v4
+; TAHITI-NEXT: v_mul_lo_u32 v13, v4, v10
+; TAHITI-NEXT: v_mul_hi_u32 v10, v4, v10
+; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, v12, v10, vcc
+; TAHITI-NEXT: v_addc_u32_e32 v11, vcc, 0, v14, vcc
+; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
+; TAHITI-NEXT: v_add_i32_e32 v3, vcc, v3, v9
+; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, v4, v10, vcc
+; TAHITI-NEXT: v_mul_lo_u32 v9, v5, v3
+; TAHITI-NEXT: v_mul_lo_u32 v10, v5, v4
; TAHITI-NEXT: v_mul_hi_u32 v5, v5, v3
-; TAHITI-NEXT: v_mul_lo_u32 v10, v10, v3
-; TAHITI-NEXT: v_mul_hi_u32 v13, v4, v11
-; TAHITI-NEXT: v_add_i32_e32 v5, vcc, v12, v5
+; TAHITI-NEXT: v_mul_lo_u32 v8, v8, v3
+; TAHITI-NEXT: v_mul_hi_u32 v11, v4, v9
; TAHITI-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; TAHITI-NEXT: v_mul_lo_u32 v10, v3, v5
-; TAHITI-NEXT: v_mul_hi_u32 v12, v3, v11
-; TAHITI-NEXT: v_mul_hi_u32 v14, v3, v5
-; TAHITI-NEXT: v_mul_lo_u32 v11, v4, v11
-; TAHITI-NEXT: v_add_i32_e32 v10, vcc, v12, v10
-; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, 0, v14, vcc
-; TAHITI-NEXT: v_mul_hi_u32 v14, v4, v5
+; TAHITI-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; TAHITI-NEXT: v_mul_lo_u32 v8, v3, v5
+; TAHITI-NEXT: v_mul_hi_u32 v10, v3, v9
+; TAHITI-NEXT: v_mul_hi_u32 v12, v3, v5
+; TAHITI-NEXT: v_mul_lo_u32 v9, v4, v9
+; TAHITI-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, 0, v12, vcc
+; TAHITI-NEXT: v_mul_hi_u32 v12, v4, v5
; TAHITI-NEXT: v_mul_lo_u32 v5, v4, v5
-; TAHITI-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, v12, v13, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v11, vcc, 0, v14, vcc
-; TAHITI-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
+; TAHITI-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; TAHITI-NEXT: v_addc_u32_e32 v8, vcc, v10, v11, vcc
+; TAHITI-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
+; TAHITI-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; TAHITI-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; TAHITI-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, v4, v10, vcc
+; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, v4, v8, vcc
; TAHITI-NEXT: v_ashrrev_i32_e32 v5, 31, v7
-; TAHITI-NEXT: v_add_i32_e32 v10, vcc, v6, v5
-; TAHITI-NEXT: v_xor_b32_e32 v10, v10, v5
-; TAHITI-NEXT: v_mul_lo_u32 v11, v10, v4
-; TAHITI-NEXT: v_mul_hi_u32 v12, v10, v3
-; TAHITI-NEXT: v_mul_hi_u32 v13, v10, v4
+; TAHITI-NEXT: v_add_i32_e32 v8, vcc, v6, v5
+; TAHITI-NEXT: v_xor_b32_e32 v8, v8, v5
+; TAHITI-NEXT: v_mul_lo_u32 v9, v8, v4
+; TAHITI-NEXT: v_mul_hi_u32 v10, v8, v3
+; TAHITI-NEXT: v_mul_hi_u32 v11, v8, v4
; TAHITI-NEXT: v_addc_u32_e32 v7, vcc, v7, v5, vcc
; TAHITI-NEXT: v_xor_b32_e32 v7, v7, v5
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; TAHITI-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
-; TAHITI-NEXT: v_mul_lo_u32 v13, v7, v3
+; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; TAHITI-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
+; TAHITI-NEXT: v_mul_lo_u32 v11, v7, v3
; TAHITI-NEXT: v_mul_hi_u32 v3, v7, v3
-; TAHITI-NEXT: v_mul_hi_u32 v14, v7, v4
+; TAHITI-NEXT: v_mul_hi_u32 v12, v7, v4
; TAHITI-NEXT: v_mul_lo_u32 v4, v7, v4
-; TAHITI-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; TAHITI-NEXT: v_addc_u32_e32 v3, vcc, v12, v3, vcc
-; TAHITI-NEXT: v_addc_u32_e32 v11, vcc, 0, v14, vcc
+; TAHITI-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; TAHITI-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc
+; TAHITI-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
; TAHITI-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, 0, v11, vcc
+; TAHITI-NEXT: v_addc_u32_e32 v4, vcc, 0, v9, vcc
; TAHITI-NEXT: v_mul_lo_u32 v4, v1, v4
-; TAHITI-NEXT: v_mul_hi_u32 v11, v1, v3
-; TAHITI-NEXT: v_mul_lo_u32 v12, v0, v3
+; TAHITI-NEXT: v_mul_hi_u32 v9, v1, v3
+; TAHITI-NEXT: v_mul_lo_u32 v10, v0, v3
; TAHITI-NEXT: v_mul_lo_u32 v3, v1, v3
-; TAHITI-NEXT: v_add_i32_e32 v4, vcc, v4, v11
-; TAHITI-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; TAHITI-NEXT: v_sub_i32_e32 v11, vcc, v7, v4
-; TAHITI-NEXT: v_sub_i32_e32 v3, vcc, v10, v3
-; TAHITI-NEXT: v_subb_u32_e64 v10, s[0:1], v11, v0, vcc
-; TAHITI-NEXT: v_sub_i32_e64 v11, s[0:1], v3, v1
-; TAHITI-NEXT: v_subbrev_u32_e64 v12, s[2:3], 0, v10, s[0:1]
-; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v12, v0
-; TAHITI-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[2:3]
-; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v11, v1
+; TAHITI-NEXT: v_add_i32_e32 v4, vcc, v4, v9
+; TAHITI-NEXT: v_add_i32_e32 v4, vcc, v10, v4
+; TAHITI-NEXT: v_sub_i32_e32 v9, vcc, v7, v4
+; TAHITI-NEXT: v_sub_i32_e32 v3, vcc, v8, v3
+; TAHITI-NEXT: v_subb_u32_e64 v8, s[0:1], v9, v0, vcc
+; TAHITI-NEXT: v_sub_i32_e64 v9, s[0:1], v3, v1
+; TAHITI-NEXT: v_subbrev_u32_e64 v10, s[2:3], 0, v8, s[0:1]
+; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v10, v0
+; TAHITI-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[2:3]
+; TAHITI-NEXT: v_cmp_ge_u32_e64 s[2:3], v9, v1
; TAHITI-NEXT: v_subb_u32_e32 v4, vcc, v7, v4, vcc
-; TAHITI-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[2:3]
-; TAHITI-NEXT: v_cmp_eq_u32_e64 s[2:3], v12, v0
-; TAHITI-NEXT: v_subb_u32_e64 v10, s[0:1], v10, v0, s[0:1]
+; TAHITI-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[2:3]
+; TAHITI-NEXT: v_cmp_eq_u32_e64 s[2:3], v10, v0
+; TAHITI-NEXT: v_subb_u32_e64 v8, s[0:1], v8, v0, s[0:1]
; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v4, v0
-; TAHITI-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[2:3]
-; TAHITI-NEXT: v_sub_i32_e64 v14, s[0:1], v11, v1
+; TAHITI-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[2:3]
+; TAHITI-NEXT: v_sub_i32_e64 v12, s[0:1], v9, v1
; TAHITI-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1
-; TAHITI-NEXT: v_subbrev_u32_e64 v10, s[0:1], 0, v10, s[0:1]
+; TAHITI-NEXT: v_subbrev_u32_e64 v8, s[0:1], 0, v8, s[0:1]
; TAHITI-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; TAHITI-NEXT: v_cmp_eq_u32_e32 vcc, v4, v0
-; TAHITI-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v13
+; TAHITI-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v11
; TAHITI-NEXT: v_cndmask_b32_e32 v0, v7, v1, vcc
-; TAHITI-NEXT: v_cndmask_b32_e64 v11, v11, v14, s[0:1]
+; TAHITI-NEXT: v_cndmask_b32_e64 v9, v9, v12, s[0:1]
; TAHITI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; TAHITI-NEXT: v_cndmask_b32_e64 v10, v12, v10, s[0:1]
-; TAHITI-NEXT: v_cndmask_b32_e32 v1, v3, v11, vcc
-; TAHITI-NEXT: v_cndmask_b32_e32 v0, v4, v10, vcc
+; TAHITI-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[0:1]
+; TAHITI-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc
+; TAHITI-NEXT: v_cndmask_b32_e32 v0, v4, v8, vcc
; TAHITI-NEXT: v_xor_b32_e32 v1, v1, v5
-; TAHITI-NEXT: v_xor_b32_e32 v0, v0, v5
-; TAHITI-NEXT: v_sub_i32_e32 v10, vcc, v1, v5
-; TAHITI-NEXT: v_subb_u32_e32 v11, vcc, v0, v5, vcc
-; TAHITI-NEXT: s_branch .LBB10_8
-; TAHITI-NEXT: .LBB10_7:
-; TAHITI-NEXT: s_mov_b64 s[6:7], -1
-; TAHITI-NEXT: .LBB10_8: ; %Flow
-; TAHITI-NEXT: s_and_b64 s[0:1], s[6:7], exec
+; TAHITI-NEXT: v_xor_b32_e32 v3, v0, v5
+; TAHITI-NEXT: v_sub_i32_e32 v0, vcc, v1, v5
+; TAHITI-NEXT: v_subb_u32_e32 v1, vcc, v3, v5, vcc
+; TAHITI-NEXT: s_branch .LBB10_3
+; TAHITI-NEXT: .LBB10_2:
+; TAHITI-NEXT: s_mov_b64 s[10:11], -1
+; TAHITI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; TAHITI-NEXT: .LBB10_3: ; %Flow5
+; TAHITI-NEXT: s_and_b64 s[0:1], s[10:11], exec
; TAHITI-NEXT: s_cselect_b32 s0, 1, 0
; TAHITI-NEXT: s_cmp_lg_u32 s0, 1
-; TAHITI-NEXT: s_cbranch_scc1 .LBB10_10
-; TAHITI-NEXT: ; %bb.9:
+; TAHITI-NEXT: s_cbranch_scc1 .LBB10_5
+; TAHITI-NEXT: ; %bb.4:
; TAHITI-NEXT: v_cvt_f32_u32_e32 v0, v2
; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, 0, v2
-; TAHITI-NEXT: v_mov_b32_e32 v11, 0
; TAHITI-NEXT: v_rcp_iflag_f32_e32 v0, v0
; TAHITI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; TAHITI-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -3348,131 +3201,472 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TAHITI-NEXT: v_mul_hi_u32 v0, v6, v0
; TAHITI-NEXT: v_mul_lo_u32 v0, v0, v2
; TAHITI-NEXT: v_sub_i32_e32 v0, vcc, v6, v0
-; TAHITI-NEXT: v_subrev_i32_e32 v1, vcc, v2, v0
+; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; TAHITI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; TAHITI-NEXT: v_subrev_i32_e32 v1, vcc, v2, v0
+; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; TAHITI-NEXT: v_cndmask_b32_e32 v10, v0, v1, vcc
+; TAHITI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; TAHITI-NEXT: v_mov_b32_e32 v1, 0
+; TAHITI-NEXT: .LBB10_5: ; %.split
+; TAHITI-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
+; TAHITI-NEXT: s_cmp_lg_u32 s1, 0
+; TAHITI-NEXT: s_cbranch_scc0 .LBB10_7
+; TAHITI-NEXT: ; %bb.6:
+; TAHITI-NEXT: s_ashr_i32 s0, s7, 31
+; TAHITI-NEXT: s_add_u32 s2, s6, s0
+; TAHITI-NEXT: s_mov_b32 s1, s0
+; TAHITI-NEXT: s_addc_u32 s3, s7, s0
+; TAHITI-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; TAHITI-NEXT: v_cvt_f32_u32_e32 v2, s0
+; TAHITI-NEXT: v_cvt_f32_u32_e32 v3, s1
+; TAHITI-NEXT: s_sub_u32 s2, 0, s0
+; TAHITI-NEXT: s_subb_u32 s3, 0, s1
+; TAHITI-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
+; TAHITI-NEXT: v_rcp_f32_e32 v2, v2
+; TAHITI-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; TAHITI-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; TAHITI-NEXT: v_trunc_f32_e32 v3, v3
+; TAHITI-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
+; TAHITI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; TAHITI-NEXT: v_cvt_u32_f32_e32 v3, v3
+; TAHITI-NEXT: v_mul_hi_u32 v4, s2, v2
+; TAHITI-NEXT: v_readfirstlane_b32 s7, v3
+; TAHITI-NEXT: v_readfirstlane_b32 s10, v2
+; TAHITI-NEXT: s_mul_i32 s11, s2, s7
+; TAHITI-NEXT: v_readfirstlane_b32 s14, v4
+; TAHITI-NEXT: s_mul_i32 s12, s3, s10
+; TAHITI-NEXT: s_mul_i32 s13, s2, s10
+; TAHITI-NEXT: s_add_i32 s11, s14, s11
+; TAHITI-NEXT: v_mul_hi_u32 v5, v2, s13
+; TAHITI-NEXT: s_add_i32 s11, s11, s12
+; TAHITI-NEXT: v_mul_hi_u32 v2, v2, s11
+; TAHITI-NEXT: v_mul_hi_u32 v6, v3, s13
+; TAHITI-NEXT: v_readfirstlane_b32 s12, v5
+; TAHITI-NEXT: s_mul_i32 s14, s10, s11
+; TAHITI-NEXT: s_add_u32 s12, s12, s14
+; TAHITI-NEXT: v_readfirstlane_b32 s14, v2
+; TAHITI-NEXT: v_mul_hi_u32 v2, v3, s11
+; TAHITI-NEXT: s_addc_u32 s14, 0, s14
+; TAHITI-NEXT: s_mul_i32 s13, s7, s13
+; TAHITI-NEXT: v_readfirstlane_b32 s15, v6
+; TAHITI-NEXT: s_add_u32 s12, s12, s13
+; TAHITI-NEXT: s_addc_u32 s12, s14, s15
+; TAHITI-NEXT: v_readfirstlane_b32 s13, v2
+; TAHITI-NEXT: s_addc_u32 s13, s13, 0
+; TAHITI-NEXT: s_mul_i32 s11, s7, s11
+; TAHITI-NEXT: s_add_u32 s11, s12, s11
+; TAHITI-NEXT: s_addc_u32 s12, 0, s13
+; TAHITI-NEXT: s_add_u32 s10, s10, s11
+; TAHITI-NEXT: v_mov_b32_e32 v3, s10
+; TAHITI-NEXT: v_mul_hi_u32 v3, s2, v3
+; TAHITI-NEXT: s_addc_u32 s7, s7, s12
+; TAHITI-NEXT: s_mul_i32 s11, s2, s10
+; TAHITI-NEXT: s_mul_i32 s2, s2, s7
+; TAHITI-NEXT: v_readfirstlane_b32 s12, v3
+; TAHITI-NEXT: s_add_i32 s2, s12, s2
+; TAHITI-NEXT: s_mul_i32 s3, s3, s10
+; TAHITI-NEXT: v_mov_b32_e32 v2, s11
+; TAHITI-NEXT: s_add_i32 s2, s2, s3
+; TAHITI-NEXT: v_mul_hi_u32 v4, s7, v2
+; TAHITI-NEXT: v_mov_b32_e32 v3, s2
+; TAHITI-NEXT: v_mul_hi_u32 v2, s10, v2
+; TAHITI-NEXT: v_mul_hi_u32 v5, s10, v3
+; TAHITI-NEXT: s_mul_i32 s12, s10, s2
+; TAHITI-NEXT: s_mul_i32 s11, s7, s11
+; TAHITI-NEXT: v_readfirstlane_b32 s14, v2
+; TAHITI-NEXT: v_mul_hi_u32 v2, s7, v3
+; TAHITI-NEXT: v_readfirstlane_b32 s13, v5
+; TAHITI-NEXT: s_add_u32 s12, s14, s12
+; TAHITI-NEXT: s_addc_u32 s13, 0, s13
+; TAHITI-NEXT: v_readfirstlane_b32 s3, v4
+; TAHITI-NEXT: s_add_u32 s11, s12, s11
+; TAHITI-NEXT: s_addc_u32 s3, s13, s3
+; TAHITI-NEXT: v_readfirstlane_b32 s11, v2
+; TAHITI-NEXT: s_addc_u32 s11, s11, 0
+; TAHITI-NEXT: s_mul_i32 s2, s7, s2
+; TAHITI-NEXT: s_add_u32 s2, s3, s2
+; TAHITI-NEXT: s_addc_u32 s3, 0, s11
+; TAHITI-NEXT: s_add_u32 s14, s10, s2
+; TAHITI-NEXT: s_addc_u32 s7, s7, s3
+; TAHITI-NEXT: s_ashr_i32 s10, s9, 31
+; TAHITI-NEXT: s_add_u32 s2, s8, s10
+; TAHITI-NEXT: s_mov_b32 s11, s10
+; TAHITI-NEXT: s_addc_u32 s3, s9, s10
+; TAHITI-NEXT: s_xor_b64 s[12:13], s[2:3], s[10:11]
+; TAHITI-NEXT: v_mov_b32_e32 v2, s7
+; TAHITI-NEXT: v_mul_hi_u32 v3, s12, v2
+; TAHITI-NEXT: v_mov_b32_e32 v4, s14
+; TAHITI-NEXT: v_mul_hi_u32 v5, s12, v4
+; TAHITI-NEXT: s_mul_i32 s2, s12, s7
+; TAHITI-NEXT: v_readfirstlane_b32 s3, v3
+; TAHITI-NEXT: v_mul_hi_u32 v3, s13, v4
+; TAHITI-NEXT: v_readfirstlane_b32 s9, v5
+; TAHITI-NEXT: v_mul_hi_u32 v2, s13, v2
+; TAHITI-NEXT: s_add_u32 s2, s9, s2
+; TAHITI-NEXT: s_addc_u32 s3, 0, s3
+; TAHITI-NEXT: s_mul_i32 s9, s13, s14
+; TAHITI-NEXT: v_readfirstlane_b32 s14, v3
+; TAHITI-NEXT: s_add_u32 s2, s2, s9
+; TAHITI-NEXT: s_addc_u32 s2, s3, s14
+; TAHITI-NEXT: v_readfirstlane_b32 s3, v2
+; TAHITI-NEXT: s_addc_u32 s9, s3, 0
+; TAHITI-NEXT: s_mul_i32 s3, s13, s7
+; TAHITI-NEXT: s_add_u32 s7, s2, s3
+; TAHITI-NEXT: v_mov_b32_e32 v2, s7
+; TAHITI-NEXT: v_mul_hi_u32 v2, s0, v2
+; TAHITI-NEXT: s_addc_u32 s9, 0, s9
+; TAHITI-NEXT: s_mul_i32 s9, s0, s9
+; TAHITI-NEXT: s_mov_b64 s[2:3], 0
+; TAHITI-NEXT: v_readfirstlane_b32 s14, v2
+; TAHITI-NEXT: s_add_i32 s9, s14, s9
+; TAHITI-NEXT: s_mul_i32 s14, s1, s7
+; TAHITI-NEXT: s_add_i32 s9, s9, s14
+; TAHITI-NEXT: s_sub_i32 s16, s13, s9
+; TAHITI-NEXT: s_mul_i32 s7, s0, s7
+; TAHITI-NEXT: s_sub_u32 s7, s12, s7
+; TAHITI-NEXT: s_cselect_b64 s[14:15], -1, 0
+; TAHITI-NEXT: s_subb_u32 s12, s16, s1
+; TAHITI-NEXT: s_sub_u32 s18, s7, s0
+; TAHITI-NEXT: s_cselect_b64 s[16:17], -1, 0
+; TAHITI-NEXT: s_subb_u32 s19, s12, 0
+; TAHITI-NEXT: s_cmp_ge_u32 s19, s1
+; TAHITI-NEXT: s_cselect_b32 s20, -1, 0
+; TAHITI-NEXT: s_cmp_ge_u32 s18, s0
+; TAHITI-NEXT: s_cselect_b32 s21, -1, 0
+; TAHITI-NEXT: s_cmp_eq_u32 s19, s1
+; TAHITI-NEXT: s_cselect_b32 s20, s21, s20
+; TAHITI-NEXT: s_or_b32 s16, s16, s17
+; TAHITI-NEXT: s_subb_u32 s12, s12, s1
+; TAHITI-NEXT: s_sub_u32 s16, s18, s0
+; TAHITI-NEXT: s_subb_u32 s12, s12, 0
+; TAHITI-NEXT: s_cmp_lg_u32 s20, 0
+; TAHITI-NEXT: s_cselect_b32 s16, s16, s18
+; TAHITI-NEXT: s_cselect_b32 s12, s12, s19
+; TAHITI-NEXT: s_or_b32 s14, s14, s15
+; TAHITI-NEXT: s_subb_u32 s9, s13, s9
+; TAHITI-NEXT: s_cmp_ge_u32 s9, s1
+; TAHITI-NEXT: s_cselect_b32 s13, -1, 0
+; TAHITI-NEXT: s_cmp_ge_u32 s7, s0
+; TAHITI-NEXT: s_cselect_b32 s0, -1, 0
+; TAHITI-NEXT: s_cmp_eq_u32 s9, s1
+; TAHITI-NEXT: s_cselect_b32 s0, s0, s13
+; TAHITI-NEXT: s_cmp_lg_u32 s0, 0
+; TAHITI-NEXT: s_cselect_b32 s1, s12, s9
+; TAHITI-NEXT: s_cselect_b32 s0, s16, s7
+; TAHITI-NEXT: s_xor_b64 s[0:1], s[0:1], s[10:11]
+; TAHITI-NEXT: s_sub_u32 s0, s0, s10
+; TAHITI-NEXT: s_subb_u32 s1, s1, s10
+; TAHITI-NEXT: s_branch .LBB10_8
+; TAHITI-NEXT: .LBB10_7:
+; TAHITI-NEXT: s_mov_b64 s[2:3], -1
+; TAHITI-NEXT: ; implicit-def: $sgpr0_sgpr1
+; TAHITI-NEXT: .LBB10_8: ; %Flow
+; TAHITI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; TAHITI-NEXT: s_cselect_b32 s2, 1, 0
+; TAHITI-NEXT: s_cmp_lg_u32 s2, 1
+; TAHITI-NEXT: s_cbranch_scc1 .LBB10_10
+; TAHITI-NEXT: ; %bb.9:
+; TAHITI-NEXT: v_cvt_f32_u32_e32 v2, s6
+; TAHITI-NEXT: s_sub_i32 s0, 0, s6
+; TAHITI-NEXT: s_mov_b32 s1, 0
+; TAHITI-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; TAHITI-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; TAHITI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; TAHITI-NEXT: v_mul_lo_u32 v3, s0, v2
+; TAHITI-NEXT: v_mul_hi_u32 v3, v2, v3
+; TAHITI-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; TAHITI-NEXT: v_mul_hi_u32 v2, s8, v2
+; TAHITI-NEXT: v_readfirstlane_b32 s0, v2
+; TAHITI-NEXT: s_mul_i32 s0, s0, s6
+; TAHITI-NEXT: s_sub_i32 s0, s8, s0
+; TAHITI-NEXT: s_sub_i32 s2, s0, s6
+; TAHITI-NEXT: s_cmp_ge_u32 s0, s6
+; TAHITI-NEXT: s_cselect_b32 s0, s2, s0
+; TAHITI-NEXT: s_sub_i32 s2, s0, s6
+; TAHITI-NEXT: s_cmp_ge_u32 s0, s6
+; TAHITI-NEXT: s_cselect_b32 s0, s2, s0
; TAHITI-NEXT: .LBB10_10: ; %.split.split
; TAHITI-NEXT: s_mov_b32 s7, 0xf000
; TAHITI-NEXT: s_mov_b32 s6, -1
-; TAHITI-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; TAHITI-NEXT: v_mov_b32_e32 v2, s0
+; TAHITI-NEXT: v_mov_b32_e32 v3, s1
+; TAHITI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; TAHITI-NEXT: s_endpgm
;
; TONGA-LABEL: srem_v2i64:
; TONGA: ; %bb.0:
-; TONGA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; TONGA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: s_add_u32 s0, s6, 16
-; TONGA-NEXT: s_addc_u32 s1, s7, 0
-; TONGA-NEXT: v_mov_b32_e32 v0, s0
-; TONGA-NEXT: v_mov_b32_e32 v1, s1
-; TONGA-NEXT: v_mov_b32_e32 v4, s6
-; TONGA-NEXT: v_mov_b32_e32 v5, s7
+; TONGA-NEXT: s_add_u32 s4, s2, 16
+; TONGA-NEXT: s_addc_u32 s5, s3, 0
+; TONGA-NEXT: v_mov_b32_e32 v0, s4
+; TONGA-NEXT: v_mov_b32_e32 v1, s5
+; TONGA-NEXT: v_mov_b32_e32 v5, s3
+; TONGA-NEXT: v_mov_b32_e32 v4, s2
; TONGA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; TONGA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; TONGA-NEXT: s_waitcnt vmcnt(1)
-; TONGA-NEXT: v_readfirstlane_b32 s1, v1
-; TONGA-NEXT: v_readfirstlane_b32 s0, v0
+; TONGA-NEXT: v_readfirstlane_b32 s7, v1
+; TONGA-NEXT: v_readfirstlane_b32 s6, v0
; TONGA-NEXT: s_waitcnt vmcnt(0)
-; TONGA-NEXT: v_readfirstlane_b32 s3, v5
-; TONGA-NEXT: v_readfirstlane_b32 s2, v4
-; TONGA-NEXT: s_or_b64 s[6:7], s[2:3], s[0:1]
-; TONGA-NEXT: s_cmp_lg_u32 s7, 0
+; TONGA-NEXT: v_readfirstlane_b32 s9, v5
+; TONGA-NEXT: v_readfirstlane_b32 s8, v4
+; TONGA-NEXT: s_or_b64 s[10:11], s[8:9], s[6:7]
+; TONGA-NEXT: v_readfirstlane_b32 s4, v6
+; TONGA-NEXT: v_readfirstlane_b32 s5, v7
+; TONGA-NEXT: v_readfirstlane_b32 s2, v2
+; TONGA-NEXT: s_cmp_lg_u32 s11, 0
+; TONGA-NEXT: v_readfirstlane_b32 s3, v3
; TONGA-NEXT: s_cbranch_scc0 .LBB10_2
; TONGA-NEXT: ; %bb.1:
-; TONGA-NEXT: s_ashr_i32 s6, s1, 31
-; TONGA-NEXT: s_add_u32 s8, s0, s6
-; TONGA-NEXT: s_mov_b32 s7, s6
-; TONGA-NEXT: s_addc_u32 s9, s1, s6
-; TONGA-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
-; TONGA-NEXT: v_cvt_f32_u32_e32 v0, s6
-; TONGA-NEXT: v_cvt_f32_u32_e32 v1, s7
-; TONGA-NEXT: s_sub_u32 s1, 0, s6
-; TONGA-NEXT: s_subb_u32 s10, 0, s7
+; TONGA-NEXT: s_ashr_i32 s10, s7, 31
+; TONGA-NEXT: s_add_u32 s12, s6, s10
+; TONGA-NEXT: s_mov_b32 s11, s10
+; TONGA-NEXT: s_addc_u32 s13, s7, s10
+; TONGA-NEXT: s_xor_b64 s[10:11], s[12:13], s[10:11]
+; TONGA-NEXT: v_cvt_f32_u32_e32 v0, s10
+; TONGA-NEXT: v_cvt_f32_u32_e32 v1, s11
+; TONGA-NEXT: s_sub_u32 s7, 0, s10
+; TONGA-NEXT: s_subb_u32 s14, 0, s11
; TONGA-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
; TONGA-NEXT: v_rcp_f32_e32 v0, v0
; TONGA-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; TONGA-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; TONGA-NEXT: v_trunc_f32_e32 v1, v1
; TONGA-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; TONGA-NEXT: v_cvt_u32_f32_e32 v10, v1
-; TONGA-NEXT: v_cvt_u32_f32_e32 v11, v0
-; TONGA-NEXT: v_mul_lo_u32 v4, s1, v10
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s1, v11, 0
-; TONGA-NEXT: v_mul_lo_u32 v5, s10, v11
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v4
-; TONGA-NEXT: v_add_u32_e32 v8, vcc, v1, v5
-; TONGA-NEXT: v_mul_hi_u32 v12, v11, v0
-; TONGA-NEXT: v_mad_u64_u32 v[4:5], s[8:9], v11, v8, 0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v10, v0, 0
-; TONGA-NEXT: v_mad_u64_u32 v[8:9], s[8:9], v10, v8, 0
-; TONGA-NEXT: v_add_u32_e32 v4, vcc, v12, v4
-; TONGA-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v4, v0
-; TONGA-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v8
+; TONGA-NEXT: v_cvt_u32_f32_e32 v7, v1
+; TONGA-NEXT: v_cvt_u32_f32_e32 v8, v0
+; TONGA-NEXT: v_mul_lo_u32 v2, s7, v7
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], s7, v8, 0
+; TONGA-NEXT: v_mul_lo_u32 v3, s14, v8
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; TONGA-NEXT: v_add_u32_e32 v5, vcc, v1, v3
+; TONGA-NEXT: v_mul_hi_u32 v9, v8, v0
+; TONGA-NEXT: v_mad_u64_u32 v[1:2], s[12:13], v8, v5, 0
+; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[12:13], v7, v0, 0
+; TONGA-NEXT: v_mad_u64_u32 v[5:6], s[12:13], v7, v5, 0
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v9, v1
+; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v3
+; TONGA-NEXT: v_addc_u32_e32 v0, vcc, v1, v4, vcc
+; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v6, vcc
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v5
; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; TONGA-NEXT: v_add_u32_e32 v8, vcc, v11, v0
-; TONGA-NEXT: v_addc_u32_e32 v9, vcc, v10, v1, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s1, v8, 0
-; TONGA-NEXT: v_mul_lo_u32 v4, s1, v9
-; TONGA-NEXT: v_mul_lo_u32 v5, s10, v8
-; TONGA-NEXT: s_ashr_i32 s10, s3, 31
-; TONGA-NEXT: v_mul_hi_u32 v11, v8, v0
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v4, v1
-; TONGA-NEXT: v_add_u32_e32 v10, vcc, v5, v1
-; TONGA-NEXT: v_mad_u64_u32 v[4:5], s[8:9], v8, v10, 0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v9, v0, 0
-; TONGA-NEXT: s_mov_b32 s11, s10
-; TONGA-NEXT: v_add_u32_e32 v11, vcc, v11, v4
-; TONGA-NEXT: v_addc_u32_e32 v12, vcc, 0, v5, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[4:5], s[8:9], v9, v10, 0
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v11, v0
-; TONGA-NEXT: v_addc_u32_e32 v0, vcc, v12, v1, vcc
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v4
+; TONGA-NEXT: v_add_u32_e32 v5, vcc, v8, v0
+; TONGA-NEXT: v_addc_u32_e32 v6, vcc, v7, v1, vcc
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], s7, v5, 0
+; TONGA-NEXT: v_mul_lo_u32 v2, s7, v6
+; TONGA-NEXT: v_mul_lo_u32 v3, s14, v5
+; TONGA-NEXT: s_ashr_i32 s14, s9, 31
+; TONGA-NEXT: v_mul_hi_u32 v8, v5, v0
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, v2, v1
+; TONGA-NEXT: v_add_u32_e32 v7, vcc, v3, v1
+; TONGA-NEXT: v_mad_u64_u32 v[1:2], s[12:13], v5, v7, 0
+; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[12:13], v6, v0, 0
+; TONGA-NEXT: s_mov_b32 s15, s14
+; TONGA-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], v6, v7, 0
+; TONGA-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; TONGA-NEXT: v_add_u32_e32 v3, vcc, v8, v3
+; TONGA-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; TONGA-NEXT: s_add_u32 s8, s2, s10
-; TONGA-NEXT: v_add_u32_e32 v4, vcc, v8, v0
-; TONGA-NEXT: s_addc_u32 s9, s3, s10
-; TONGA-NEXT: v_addc_u32_e32 v5, vcc, v9, v1, vcc
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v2, v0
+; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; TONGA-NEXT: s_add_u32 s12, s8, s14
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v5, v0
+; TONGA-NEXT: s_addc_u32 s13, s9, s14
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, v6, v1, vcc
+; TONGA-NEXT: s_xor_b64 s[16:17], s[12:13], s[14:15]
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], s16, v3, 0
+; TONGA-NEXT: v_mul_hi_u32 v4, s16, v2
+; TONGA-NEXT: v_readfirstlane_b32 s7, v1
+; TONGA-NEXT: v_readfirstlane_b32 s9, v0
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], s17, v2, 0
+; TONGA-NEXT: v_readfirstlane_b32 s12, v4
+; TONGA-NEXT: s_add_u32 s9, s12, s9
+; TONGA-NEXT: s_addc_u32 s7, 0, s7
+; TONGA-NEXT: v_readfirstlane_b32 s18, v1
+; TONGA-NEXT: v_mad_u64_u32 v[1:2], s[12:13], s17, v3, 0
+; TONGA-NEXT: v_readfirstlane_b32 s12, v0
+; TONGA-NEXT: s_add_u32 s9, s9, s12
+; TONGA-NEXT: s_addc_u32 s7, s7, s18
+; TONGA-NEXT: v_readfirstlane_b32 s9, v2
+; TONGA-NEXT: s_addc_u32 s9, s9, 0
+; TONGA-NEXT: v_readfirstlane_b32 s12, v1
+; TONGA-NEXT: s_add_u32 s7, s7, s12
+; TONGA-NEXT: v_mov_b32_e32 v0, s7
+; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[12:13], s10, v0, 0
+; TONGA-NEXT: s_addc_u32 s9, 0, s9
+; TONGA-NEXT: s_mul_i32 s9, s10, s9
+; TONGA-NEXT: s_mul_i32 s7, s11, s7
+; TONGA-NEXT: v_readfirstlane_b32 s18, v1
+; TONGA-NEXT: s_add_i32 s9, s18, s9
+; TONGA-NEXT: s_add_i32 s9, s9, s7
+; TONGA-NEXT: s_sub_i32 s7, s17, s9
+; TONGA-NEXT: v_readfirstlane_b32 s18, v0
+; TONGA-NEXT: s_sub_u32 s16, s16, s18
+; TONGA-NEXT: s_cselect_b64 s[18:19], -1, 0
+; TONGA-NEXT: s_subb_u32 s7, s7, s11
+; TONGA-NEXT: s_sub_u32 s22, s16, s10
+; TONGA-NEXT: s_cselect_b64 s[20:21], -1, 0
+; TONGA-NEXT: s_subb_u32 s23, s7, 0
+; TONGA-NEXT: s_cmp_ge_u32 s23, s11
+; TONGA-NEXT: s_cselect_b32 s24, -1, 0
+; TONGA-NEXT: s_cmp_ge_u32 s22, s10
+; TONGA-NEXT: s_cselect_b32 s25, -1, 0
+; TONGA-NEXT: s_cmp_eq_u32 s23, s11
+; TONGA-NEXT: s_cselect_b32 s24, s25, s24
+; TONGA-NEXT: s_cmp_lg_u64 s[20:21], 0
+; TONGA-NEXT: s_subb_u32 s7, s7, s11
+; TONGA-NEXT: s_sub_u32 s20, s22, s10
+; TONGA-NEXT: s_subb_u32 s7, s7, 0
+; TONGA-NEXT: s_cmp_lg_u32 s24, 0
+; TONGA-NEXT: s_cselect_b32 s20, s20, s22
+; TONGA-NEXT: s_cselect_b32 s7, s7, s23
+; TONGA-NEXT: s_cmp_lg_u64 s[18:19], 0
+; TONGA-NEXT: s_subb_u32 s9, s17, s9
+; TONGA-NEXT: s_cmp_ge_u32 s9, s11
+; TONGA-NEXT: s_cselect_b32 s17, -1, 0
+; TONGA-NEXT: s_cmp_ge_u32 s16, s10
+; TONGA-NEXT: s_cselect_b32 s10, -1, 0
+; TONGA-NEXT: s_cmp_eq_u32 s9, s11
+; TONGA-NEXT: s_cselect_b32 s10, s10, s17
+; TONGA-NEXT: s_cmp_lg_u32 s10, 0
+; TONGA-NEXT: s_cselect_b32 s11, s7, s9
+; TONGA-NEXT: s_cselect_b32 s10, s20, s16
+; TONGA-NEXT: s_xor_b64 s[10:11], s[10:11], s[14:15]
+; TONGA-NEXT: s_sub_u32 s10, s10, s14
+; TONGA-NEXT: s_mov_b64 s[12:13], 0
+; TONGA-NEXT: s_subb_u32 s11, s11, s14
+; TONGA-NEXT: s_branch .LBB10_3
+; TONGA-NEXT: .LBB10_2:
+; TONGA-NEXT: s_mov_b64 s[12:13], -1
+; TONGA-NEXT: ; implicit-def: $sgpr10_sgpr11
+; TONGA-NEXT: .LBB10_3: ; %Flow5
+; TONGA-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; TONGA-NEXT: s_cselect_b32 s7, 1, 0
+; TONGA-NEXT: s_cmp_lg_u32 s7, 1
+; TONGA-NEXT: s_cbranch_scc1 .LBB10_5
+; TONGA-NEXT: ; %bb.4:
+; TONGA-NEXT: v_cvt_f32_u32_e32 v0, s6
+; TONGA-NEXT: s_sub_i32 s7, 0, s6
+; TONGA-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; TONGA-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; TONGA-NEXT: v_cvt_u32_f32_e32 v0, v0
+; TONGA-NEXT: v_mul_lo_u32 v1, s7, v0
+; TONGA-NEXT: v_mul_hi_u32 v1, v0, v1
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT: v_mul_hi_u32 v0, s8, v0
+; TONGA-NEXT: v_mul_lo_u32 v0, v0, s6
+; TONGA-NEXT: v_sub_u32_e32 v0, vcc, s8, v0
+; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, s6, v0
+; TONGA-NEXT: v_cmp_le_u32_e32 vcc, s6, v0
+; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, s6, v0
+; TONGA-NEXT: v_cmp_le_u32_e32 vcc, s6, v0
+; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; TONGA-NEXT: v_mov_b32_e32 v1, 0
+; TONGA-NEXT: s_branch .LBB10_6
+; TONGA-NEXT: .LBB10_5:
+; TONGA-NEXT: v_mov_b32_e32 v0, s10
+; TONGA-NEXT: v_mov_b32_e32 v1, s11
+; TONGA-NEXT: .LBB10_6: ; %.split
+; TONGA-NEXT: s_or_b64 s[6:7], s[4:5], s[2:3]
+; TONGA-NEXT: s_cmp_lg_u32 s7, 0
+; TONGA-NEXT: s_cbranch_scc0 .LBB10_8
+; TONGA-NEXT: ; %bb.7:
+; TONGA-NEXT: s_ashr_i32 s6, s3, 31
+; TONGA-NEXT: s_add_u32 s8, s2, s6
+; TONGA-NEXT: s_mov_b32 s7, s6
+; TONGA-NEXT: s_addc_u32 s9, s3, s6
+; TONGA-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
+; TONGA-NEXT: v_cvt_f32_u32_e32 v2, s6
+; TONGA-NEXT: v_cvt_f32_u32_e32 v3, s7
+; TONGA-NEXT: s_sub_u32 s3, 0, s6
+; TONGA-NEXT: s_subb_u32 s10, 0, s7
+; TONGA-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
+; TONGA-NEXT: v_rcp_f32_e32 v2, v2
+; TONGA-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; TONGA-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; TONGA-NEXT: v_trunc_f32_e32 v3, v3
+; TONGA-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
+; TONGA-NEXT: v_cvt_u32_f32_e32 v9, v3
+; TONGA-NEXT: v_cvt_u32_f32_e32 v10, v2
+; TONGA-NEXT: v_mul_lo_u32 v4, s3, v9
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s3, v10, 0
+; TONGA-NEXT: v_mul_lo_u32 v5, s10, v10
+; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v4
+; TONGA-NEXT: v_add_u32_e32 v7, vcc, v3, v5
+; TONGA-NEXT: v_mul_hi_u32 v11, v10, v2
+; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[8:9], v10, v7, 0
+; TONGA-NEXT: v_mad_u64_u32 v[5:6], s[8:9], v9, v2, 0
+; TONGA-NEXT: v_mad_u64_u32 v[7:8], s[8:9], v9, v7, 0
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v11, v3
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v5
+; TONGA-NEXT: v_addc_u32_e32 v2, vcc, v3, v6, vcc
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v8, vcc
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v7
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; TONGA-NEXT: v_add_u32_e32 v7, vcc, v10, v2
+; TONGA-NEXT: v_addc_u32_e32 v8, vcc, v9, v3, vcc
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s3, v7, 0
+; TONGA-NEXT: v_mul_lo_u32 v4, s3, v8
+; TONGA-NEXT: v_mul_lo_u32 v5, s10, v7
+; TONGA-NEXT: s_ashr_i32 s10, s5, 31
+; TONGA-NEXT: v_mul_hi_u32 v10, v7, v2
+; TONGA-NEXT: v_add_u32_e32 v3, vcc, v4, v3
+; TONGA-NEXT: v_add_u32_e32 v9, vcc, v5, v3
+; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[8:9], v7, v9, 0
+; TONGA-NEXT: v_mad_u64_u32 v[5:6], s[8:9], v8, v2, 0
+; TONGA-NEXT: s_mov_b32 s11, s10
+; TONGA-NEXT: v_add_u32_e32 v10, vcc, v10, v3
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v8, v9, 0
+; TONGA-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; TONGA-NEXT: v_add_u32_e32 v5, vcc, v10, v5
+; TONGA-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v4, v2
+; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; TONGA-NEXT: s_add_u32 s8, s4, s10
+; TONGA-NEXT: v_add_u32_e32 v4, vcc, v7, v2
+; TONGA-NEXT: s_addc_u32 s9, s5, s10
+; TONGA-NEXT: v_addc_u32_e32 v5, vcc, v8, v3, vcc
; TONGA-NEXT: s_xor_b64 s[12:13], s[8:9], s[10:11]
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s12, v5, 0
-; TONGA-NEXT: v_mul_hi_u32 v8, s12, v4
-; TONGA-NEXT: v_readfirstlane_b32 s1, v1
-; TONGA-NEXT: v_readfirstlane_b32 s3, v0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s13, v4, 0
-; TONGA-NEXT: v_readfirstlane_b32 s8, v8
-; TONGA-NEXT: s_add_u32 s3, s8, s3
-; TONGA-NEXT: v_mad_u64_u32 v[4:5], s[8:9], s13, v5, 0
-; TONGA-NEXT: s_addc_u32 s1, 0, s1
-; TONGA-NEXT: v_readfirstlane_b32 s8, v0
-; TONGA-NEXT: v_readfirstlane_b32 s14, v1
-; TONGA-NEXT: s_add_u32 s3, s3, s8
-; TONGA-NEXT: s_addc_u32 s1, s1, s14
-; TONGA-NEXT: v_readfirstlane_b32 s3, v5
-; TONGA-NEXT: s_addc_u32 s3, s3, 0
-; TONGA-NEXT: v_readfirstlane_b32 s8, v4
-; TONGA-NEXT: s_add_u32 s1, s1, s8
-; TONGA-NEXT: v_mov_b32_e32 v0, s1
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s6, v0, 0
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s12, v5, 0
+; TONGA-NEXT: v_mul_hi_u32 v6, s12, v4
+; TONGA-NEXT: v_readfirstlane_b32 s3, v3
+; TONGA-NEXT: v_readfirstlane_b32 s5, v2
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s13, v4, 0
+; TONGA-NEXT: v_readfirstlane_b32 s8, v6
+; TONGA-NEXT: s_add_u32 s5, s8, s5
; TONGA-NEXT: s_addc_u32 s3, 0, s3
-; TONGA-NEXT: s_mul_i32 s3, s6, s3
-; TONGA-NEXT: s_mul_i32 s1, s7, s1
-; TONGA-NEXT: v_readfirstlane_b32 s14, v1
-; TONGA-NEXT: s_add_i32 s3, s14, s3
-; TONGA-NEXT: s_add_i32 s3, s3, s1
-; TONGA-NEXT: s_sub_i32 s1, s13, s3
-; TONGA-NEXT: v_readfirstlane_b32 s14, v0
+; TONGA-NEXT: v_readfirstlane_b32 s14, v3
+; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[8:9], s13, v5, 0
+; TONGA-NEXT: v_readfirstlane_b32 s8, v2
+; TONGA-NEXT: s_add_u32 s5, s5, s8
+; TONGA-NEXT: s_addc_u32 s3, s3, s14
+; TONGA-NEXT: v_readfirstlane_b32 s5, v4
+; TONGA-NEXT: s_addc_u32 s5, s5, 0
+; TONGA-NEXT: v_readfirstlane_b32 s8, v3
+; TONGA-NEXT: s_add_u32 s3, s3, s8
+; TONGA-NEXT: v_mov_b32_e32 v2, s3
+; TONGA-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s6, v2, 0
+; TONGA-NEXT: s_addc_u32 s5, 0, s5
+; TONGA-NEXT: s_mul_i32 s5, s6, s5
+; TONGA-NEXT: s_mul_i32 s3, s7, s3
+; TONGA-NEXT: v_readfirstlane_b32 s14, v3
+; TONGA-NEXT: s_add_i32 s5, s14, s5
+; TONGA-NEXT: s_add_i32 s5, s5, s3
+; TONGA-NEXT: s_sub_i32 s3, s13, s5
+; TONGA-NEXT: v_readfirstlane_b32 s14, v2
; TONGA-NEXT: s_sub_u32 s12, s12, s14
; TONGA-NEXT: s_cselect_b64 s[14:15], -1, 0
-; TONGA-NEXT: s_subb_u32 s1, s1, s7
+; TONGA-NEXT: s_subb_u32 s3, s3, s7
; TONGA-NEXT: s_sub_u32 s18, s12, s6
; TONGA-NEXT: s_cselect_b64 s[16:17], -1, 0
-; TONGA-NEXT: s_subb_u32 s19, s1, 0
+; TONGA-NEXT: s_subb_u32 s19, s3, 0
; TONGA-NEXT: s_cmp_ge_u32 s19, s7
; TONGA-NEXT: s_cselect_b32 s20, -1, 0
; TONGA-NEXT: s_cmp_ge_u32 s18, s6
@@ -3480,201 +3674,62 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: s_cmp_eq_u32 s19, s7
; TONGA-NEXT: s_cselect_b32 s20, s21, s20
; TONGA-NEXT: s_cmp_lg_u64 s[16:17], 0
-; TONGA-NEXT: s_subb_u32 s1, s1, s7
+; TONGA-NEXT: s_subb_u32 s3, s3, s7
; TONGA-NEXT: s_sub_u32 s16, s18, s6
-; TONGA-NEXT: s_subb_u32 s1, s1, 0
+; TONGA-NEXT: s_subb_u32 s3, s3, 0
; TONGA-NEXT: s_cmp_lg_u32 s20, 0
; TONGA-NEXT: s_cselect_b32 s16, s16, s18
-; TONGA-NEXT: s_cselect_b32 s1, s1, s19
+; TONGA-NEXT: s_cselect_b32 s3, s3, s19
; TONGA-NEXT: s_cmp_lg_u64 s[14:15], 0
-; TONGA-NEXT: s_subb_u32 s3, s13, s3
-; TONGA-NEXT: s_cmp_ge_u32 s3, s7
+; TONGA-NEXT: s_subb_u32 s5, s13, s5
+; TONGA-NEXT: s_cmp_ge_u32 s5, s7
; TONGA-NEXT: s_cselect_b32 s13, -1, 0
; TONGA-NEXT: s_cmp_ge_u32 s12, s6
; TONGA-NEXT: s_cselect_b32 s6, -1, 0
-; TONGA-NEXT: s_cmp_eq_u32 s3, s7
+; TONGA-NEXT: s_cmp_eq_u32 s5, s7
; TONGA-NEXT: s_cselect_b32 s6, s6, s13
; TONGA-NEXT: s_cmp_lg_u32 s6, 0
-; TONGA-NEXT: s_cselect_b32 s7, s1, s3
+; TONGA-NEXT: s_cselect_b32 s7, s3, s5
; TONGA-NEXT: s_cselect_b32 s6, s16, s12
; TONGA-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
; TONGA-NEXT: s_sub_u32 s6, s6, s10
; TONGA-NEXT: s_mov_b64 s[8:9], 0
; TONGA-NEXT: s_subb_u32 s7, s7, s10
-; TONGA-NEXT: s_branch .LBB10_3
-; TONGA-NEXT: .LBB10_2:
-; TONGA-NEXT: s_mov_b64 s[8:9], -1
-; TONGA-NEXT: ; implicit-def: $sgpr6_sgpr7
-; TONGA-NEXT: .LBB10_3: ; %Flow5
-; TONGA-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; TONGA-NEXT: s_cselect_b32 s1, 1, 0
-; TONGA-NEXT: s_cmp_lg_u32 s1, 1
-; TONGA-NEXT: s_cbranch_scc1 .LBB10_5
-; TONGA-NEXT: ; %bb.4:
-; TONGA-NEXT: v_cvt_f32_u32_e32 v0, s0
-; TONGA-NEXT: s_sub_i32 s1, 0, s0
-; TONGA-NEXT: v_mov_b32_e32 v9, 0
-; TONGA-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; TONGA-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; TONGA-NEXT: v_cvt_u32_f32_e32 v0, v0
-; TONGA-NEXT: v_mul_lo_u32 v1, s1, v0
-; TONGA-NEXT: v_mul_hi_u32 v1, v0, v1
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; TONGA-NEXT: v_mul_hi_u32 v0, s2, v0
-; TONGA-NEXT: v_mul_lo_u32 v0, v0, s0
-; TONGA-NEXT: v_sub_u32_e32 v0, vcc, s2, v0
-; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, s0, v0
-; TONGA-NEXT: v_cmp_le_u32_e32 vcc, s0, v0
-; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, s0, v0
-; TONGA-NEXT: v_cmp_le_u32_e32 vcc, s0, v0
-; TONGA-NEXT: v_cndmask_b32_e32 v8, v0, v1, vcc
-; TONGA-NEXT: s_branch .LBB10_6
-; TONGA-NEXT: .LBB10_5:
-; TONGA-NEXT: v_mov_b32_e32 v9, s7
-; TONGA-NEXT: v_mov_b32_e32 v8, s6
-; TONGA-NEXT: .LBB10_6: ; %.split
-; TONGA-NEXT: v_or_b32_e32 v0, v7, v3
-; TONGA-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; TONGA-NEXT: s_cbranch_vccz .LBB10_8
-; TONGA-NEXT: ; %bb.7:
-; TONGA-NEXT: v_ashrrev_i32_e32 v0, 31, v3
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v2, v0
-; TONGA-NEXT: v_addc_u32_e32 v3, vcc, v3, v0, vcc
-; TONGA-NEXT: v_xor_b32_e32 v5, v1, v0
-; TONGA-NEXT: v_xor_b32_e32 v10, v3, v0
-; TONGA-NEXT: v_cvt_f32_u32_e32 v0, v5
-; TONGA-NEXT: v_cvt_f32_u32_e32 v1, v10
-; TONGA-NEXT: v_sub_u32_e32 v13, vcc, 0, v5
-; TONGA-NEXT: v_subb_u32_e32 v14, vcc, 0, v10, vcc
-; TONGA-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; TONGA-NEXT: v_rcp_f32_e32 v0, v0
-; TONGA-NEXT: s_mov_b64 s[6:7], 0
-; TONGA-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; TONGA-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; TONGA-NEXT: v_trunc_f32_e32 v1, v1
-; TONGA-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; TONGA-NEXT: v_cvt_u32_f32_e32 v11, v1
-; TONGA-NEXT: v_cvt_u32_f32_e32 v12, v0
-; TONGA-NEXT: v_mul_lo_u32 v3, v13, v11
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v13, v12, 0
-; TONGA-NEXT: v_mul_lo_u32 v4, v14, v12
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v3
-; TONGA-NEXT: v_add_u32_e32 v15, vcc, v1, v4
-; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v12, v15, 0
-; TONGA-NEXT: v_mul_hi_u32 v16, v12, v0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v11, v0, 0
-; TONGA-NEXT: v_add_u32_e32 v16, vcc, v16, v3
-; TONGA-NEXT: v_addc_u32_e32 v17, vcc, 0, v4, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v11, v15, 0
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v16, v0
-; TONGA-NEXT: v_addc_u32_e32 v0, vcc, v17, v1, vcc
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v4, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v3
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; TONGA-NEXT: v_add_u32_e32 v12, vcc, v12, v0
-; TONGA-NEXT: v_addc_u32_e32 v11, vcc, v11, v1, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v13, v12, 0
-; TONGA-NEXT: v_mul_lo_u32 v3, v13, v11
-; TONGA-NEXT: v_mul_lo_u32 v4, v14, v12
-; TONGA-NEXT: v_mul_hi_u32 v14, v12, v0
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v3, v1
-; TONGA-NEXT: v_add_u32_e32 v13, vcc, v4, v1
-; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v12, v13, 0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v11, v0, 0
-; TONGA-NEXT: v_add_u32_e32 v14, vcc, v14, v3
-; TONGA-NEXT: v_addc_u32_e32 v15, vcc, 0, v4, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v11, v13, 0
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v14, v0
-; TONGA-NEXT: v_addc_u32_e32 v0, vcc, v15, v1, vcc
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v4, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v3
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; TONGA-NEXT: v_add_u32_e32 v3, vcc, v12, v0
-; TONGA-NEXT: v_addc_u32_e32 v11, vcc, v11, v1, vcc
-; TONGA-NEXT: v_ashrrev_i32_e32 v12, 31, v7
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v6, v12
-; TONGA-NEXT: v_addc_u32_e32 v4, vcc, v7, v12, vcc
-; TONGA-NEXT: v_xor_b32_e32 v7, v0, v12
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v7, v11, 0
-; TONGA-NEXT: v_mul_hi_u32 v13, v7, v3
-; TONGA-NEXT: v_xor_b32_e32 v14, v4, v12
-; TONGA-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v14, v3, 0
-; TONGA-NEXT: v_add_u32_e32 v13, vcc, v13, v0
-; TONGA-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v14, v11, 0
-; TONGA-NEXT: v_add_u32_e32 v3, vcc, v13, v3
-; TONGA-NEXT: v_addc_u32_e32 v3, vcc, v15, v4, vcc
-; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v0
-; TONGA-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; TONGA-NEXT: v_mul_lo_u32 v4, v5, v0
-; TONGA-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v5, v3, 0
-; TONGA-NEXT: v_mul_lo_u32 v3, v10, v3
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v4, v1
-; TONGA-NEXT: v_add_u32_e32 v1, vcc, v3, v1
-; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v14, v1
-; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v7, v0
-; TONGA-NEXT: v_subb_u32_e64 v3, s[0:1], v3, v10, vcc
-; TONGA-NEXT: v_sub_u32_e64 v4, s[0:1], v0, v5
-; TONGA-NEXT: v_subbrev_u32_e64 v7, s[2:3], 0, v3, s[0:1]
-; TONGA-NEXT: v_cmp_ge_u32_e64 s[2:3], v7, v10
-; TONGA-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[2:3]
-; TONGA-NEXT: v_cmp_ge_u32_e64 s[2:3], v4, v5
-; TONGA-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[2:3]
-; TONGA-NEXT: v_cmp_eq_u32_e64 s[2:3], v7, v10
-; TONGA-NEXT: v_subb_u32_e64 v3, s[0:1], v3, v10, s[0:1]
-; TONGA-NEXT: v_cndmask_b32_e64 v11, v11, v13, s[2:3]
-; TONGA-NEXT: v_sub_u32_e64 v13, s[0:1], v4, v5
-; TONGA-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1]
-; TONGA-NEXT: v_subb_u32_e32 v1, vcc, v14, v1, vcc
-; TONGA-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v11
-; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v10
-; TONGA-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[0:1]
-; TONGA-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
-; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
-; TONGA-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; TONGA-NEXT: v_cmp_eq_u32_e32 vcc, v1, v10
-; TONGA-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; TONGA-NEXT: v_cndmask_b32_e64 v4, v4, v13, s[0:1]
-; TONGA-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; TONGA-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; TONGA-NEXT: v_xor_b32_e32 v0, v0, v12
-; TONGA-NEXT: v_xor_b32_e32 v1, v1, v12
-; TONGA-NEXT: v_sub_u32_e32 v10, vcc, v0, v12
-; TONGA-NEXT: v_subb_u32_e32 v11, vcc, v1, v12, vcc
; TONGA-NEXT: s_branch .LBB10_9
; TONGA-NEXT: .LBB10_8:
-; TONGA-NEXT: s_mov_b64 s[6:7], -1
+; TONGA-NEXT: s_mov_b64 s[8:9], -1
+; TONGA-NEXT: ; implicit-def: $sgpr6_sgpr7
; TONGA-NEXT: .LBB10_9: ; %Flow
-; TONGA-NEXT: s_and_b64 s[0:1], s[6:7], exec
-; TONGA-NEXT: s_cselect_b32 s0, 1, 0
-; TONGA-NEXT: s_cmp_lg_u32 s0, 1
+; TONGA-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; TONGA-NEXT: s_cselect_b32 s3, 1, 0
+; TONGA-NEXT: s_cmp_lg_u32 s3, 1
; TONGA-NEXT: s_cbranch_scc1 .LBB10_11
; TONGA-NEXT: ; %bb.10:
-; TONGA-NEXT: v_cvt_f32_u32_e32 v0, v2
-; TONGA-NEXT: v_sub_u32_e32 v1, vcc, 0, v2
-; TONGA-NEXT: v_mov_b32_e32 v11, 0
-; TONGA-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; TONGA-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; TONGA-NEXT: v_cvt_u32_f32_e32 v0, v0
-; TONGA-NEXT: v_mul_lo_u32 v1, v1, v0
-; TONGA-NEXT: v_mul_hi_u32 v1, v0, v1
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; TONGA-NEXT: v_mul_hi_u32 v0, v6, v0
-; TONGA-NEXT: v_mul_lo_u32 v0, v0, v2
-; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v6, v0
-; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, v2, v0
-; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, v2, v0
-; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; TONGA-NEXT: v_cndmask_b32_e32 v10, v0, v1, vcc
+; TONGA-NEXT: v_cvt_f32_u32_e32 v2, s2
+; TONGA-NEXT: s_sub_i32 s3, 0, s2
+; TONGA-NEXT: s_mov_b32 s7, 0
+; TONGA-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; TONGA-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; TONGA-NEXT: v_cvt_u32_f32_e32 v2, v2
+; TONGA-NEXT: v_mul_lo_u32 v3, s3, v2
+; TONGA-NEXT: v_mul_hi_u32 v3, v2, v3
+; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v3
+; TONGA-NEXT: v_mul_hi_u32 v2, s4, v2
+; TONGA-NEXT: v_readfirstlane_b32 s3, v2
+; TONGA-NEXT: s_mul_i32 s3, s3, s2
+; TONGA-NEXT: s_sub_i32 s3, s4, s3
+; TONGA-NEXT: s_sub_i32 s4, s3, s2
+; TONGA-NEXT: s_cmp_ge_u32 s3, s2
+; TONGA-NEXT: s_cselect_b32 s3, s4, s3
+; TONGA-NEXT: s_sub_i32 s4, s3, s2
+; TONGA-NEXT: s_cmp_ge_u32 s3, s2
+; TONGA-NEXT: s_cselect_b32 s6, s4, s3
; TONGA-NEXT: .LBB10_11: ; %.split.split
-; TONGA-NEXT: v_mov_b32_e32 v0, s4
-; TONGA-NEXT: v_mov_b32_e32 v1, s5
-; TONGA-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; TONGA-NEXT: v_mov_b32_e32 v4, s0
+; TONGA-NEXT: v_mov_b32_e32 v5, s1
+; TONGA-NEXT: v_mov_b32_e32 v2, s6
+; TONGA-NEXT: v_mov_b32_e32 v3, s7
+; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; TONGA-NEXT: s_endpgm
;
; EG-LABEL: srem_v2i64:
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
index 4a31d189887c0..1377dae8980f1 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
@@ -35,15 +35,15 @@ define amdgpu_kernel void @store_lds_v4i32(ptr addrspace(3) %out, <4 x i32> %x)
;
; GFX6-LABEL: store_lds_v4i32:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
-; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: v_mov_b32_e32 v0, s2
; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: v_mov_b32_e32 v4, s4
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_write2_b64 v4, v[2:3], v[0:1] offset1:1
; GFX6-NEXT: s_endpgm
;
@@ -543,13 +543,13 @@ define amdgpu_kernel void @store_lds_v4i32_align8(ptr addrspace(3) %out, <4 x i3
;
; GFX6-LABEL: store_lds_v4i32_align8:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
-; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_mov_b32_e32 v4, s4
; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
@@ -558,12 +558,12 @@ define amdgpu_kernel void @store_lds_v4i32_align8(ptr addrspace(3) %out, <4 x i3
; GFX10-LABEL: store_lds_v4i32_align8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v4, s6
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v4, s6
; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
@@ -614,15 +614,15 @@ define amdgpu_kernel void @store_lds_v4i32_align16(ptr addrspace(3) %out, <4 x i
;
; GFX6-LABEL: store_lds_v4i32_align16:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
-; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: v_mov_b32_e32 v0, s2
; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: v_mov_b32_e32 v4, s4
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_write2_b64 v4, v[2:3], v[0:1] offset1:1
; GFX6-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/store-local.96.ll
index ebea1d1e756b5..9a3bd06af20b3 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.96.ll
@@ -8,29 +8,26 @@
define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX9-LABEL: store_lds_v3i32:
; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX9-NEXT: ; kill: killed $sgpr4_sgpr5
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: ds_write_b96 v3, v[0:2]
; GFX9-NEXT: s_endpgm
;
; GFX7-LABEL: store_lds_v3i32:
; GFX7: ; %bb.0:
+; GFX7-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX7-NEXT: s_mov_b32 m0, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v3, s6
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
; GFX7-NEXT: ds_write_b96 v3, v[0:2]
; GFX7-NEXT: s_endpgm
;
@@ -41,35 +38,34 @@ define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x)
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: ds_write_b32 v2, v1 offset:8
; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: ds_write_b32 v2, v3 offset:8
; GFX6-NEXT: ds_write_b64 v2, v[0:1]
; GFX6-NEXT: s_endpgm
;
; GFX10-LABEL: store_lds_v3i32:
; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX10-NEXT: ; kill: killed $sgpr4_sgpr5
+; GFX10-NEXT: v_mov_b32_e32 v3, s6
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: ds_write_b96 v3, v[0:2]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: store_lds_v3i32:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v0, s0
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out
@@ -448,11 +444,11 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: ds_write_b32 v2, v3 offset:8
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
+; GFX9-NEXT: ds_write_b32 v2, v3 offset:8
; GFX9-NEXT: s_endpgm
;
; GFX7-LABEL: store_lds_v3i32_align8:
@@ -462,11 +458,11 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: ds_write_b32 v2, v1 offset:8
; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: v_mov_b32_e32 v3, s2
; GFX7-NEXT: ds_write_b64 v2, v[0:1]
+; GFX7-NEXT: ds_write_b32 v2, v3 offset:8
; GFX7-NEXT: s_endpgm
;
; GFX6-LABEL: store_lds_v3i32_align8:
@@ -476,11 +472,11 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: ds_write_b32 v2, v1 offset:8
; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: ds_write_b32 v2, v3 offset:8
; GFX6-NEXT: s_endpgm
;
; GFX10-LABEL: store_lds_v3i32_align8:
@@ -490,11 +486,11 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s6
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: ds_write_b32 v2, v3 offset:8
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: ds_write_b64 v2, v[0:1]
+; GFX10-NEXT: ds_write_b32 v2, v3 offset:8
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: store_lds_v3i32_align8:
@@ -503,10 +499,10 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: ds_store_b32 v2, v3 offset:8
+; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: ds_store_b64 v2, v[0:1]
+; GFX11-NEXT: ds_store_b32 v2, v3 offset:8
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 8
ret void
@@ -515,29 +511,26 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
define amdgpu_kernel void @store_lds_v3i32_align16(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX9-LABEL: store_lds_v3i32_align16:
; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX9-NEXT: ; kill: killed $sgpr4_sgpr5
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: ds_write_b96 v3, v[0:2]
; GFX9-NEXT: s_endpgm
;
; GFX7-LABEL: store_lds_v3i32_align16:
; GFX7: ; %bb.0:
+; GFX7-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX7-NEXT: s_mov_b32 m0, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v3, s6
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
; GFX7-NEXT: ds_write_b96 v3, v[0:2]
; GFX7-NEXT: s_endpgm
;
@@ -548,35 +541,34 @@ define amdgpu_kernel void @store_lds_v3i32_align16(ptr addrspace(3) %out, <3 x i
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: ds_write_b32 v2, v1 offset:8
; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: ds_write_b32 v2, v3 offset:8
; GFX6-NEXT: ds_write_b64 v2, v[0:1]
; GFX6-NEXT: s_endpgm
;
; GFX10-LABEL: store_lds_v3i32_align16:
; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX10-NEXT: ; kill: killed $sgpr4_sgpr5
+; GFX10-NEXT: v_mov_b32_e32 v3, s6
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: ds_write_b96 v3, v[0:2]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: store_lds_v3i32_align16:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v0, s0
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
index de7d4817ebaa9..000131f4986dd 100644
--- a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
@@ -52,8 +52,7 @@ define amdgpu_kernel void @store_as4_i64(ptr addrspace(4) %p, i64 %v) {
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; CHECK-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; CHECK-NEXT: s_endpgm
store i64 %v, ptr addrspace(4) %p
@@ -80,8 +79,7 @@ define amdgpu_kernel void @store_as4_double(ptr addrspace(4) %p, double %v) {
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v2, s2
-; CHECK-NEXT: v_mov_b32_e32 v3, s3
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; CHECK-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; CHECK-NEXT: s_endpgm
store double %v, ptr addrspace(4) %p
diff --git a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
index 485a2e2dac300..ee6a652477c69 100644
--- a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
+++ b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
@@ -11,14 +11,14 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; CHECK-NEXT: flat_load_dword v42, v[44:45]
; CHECK-NEXT: s_mov_b64 s[34:35], s[8:9]
-; CHECK-NEXT: s_load_dwordx4 s[64:67], s[34:35], 0x8
; CHECK-NEXT: s_load_dword s68, s[34:35], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[64:67], s[34:35], 0x8
; CHECK-NEXT: s_add_u32 s0, s0, s17
; CHECK-NEXT: s_addc_u32 s1, s1, 0
; CHECK-NEXT: s_mov_b64 s[8:9], src_private_base
-; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_cmp_lg_u32 s68, -1
+; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
; CHECK-NEXT: s_mov_b64 s[48:49], s[4:5]
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: s_cselect_b32 s5, s9, 0
@@ -27,7 +27,7 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: v_mov_b32_e32 v47, s5
; CHECK-NEXT: s_mov_b32 s5, s4
; CHECK-NEXT: s_addc_u32 s51, s35, 0
-; CHECK-NEXT: v_pk_mov_b32 v[62:63], s[4:5], s[4:5] op_sel:[0,1]
+; CHECK-NEXT: v_pk_mov_b32 v[60:61], s[4:5], s[4:5] op_sel:[0,1]
; CHECK-NEXT: s_getpc_b64 s[4:5]
; CHECK-NEXT: s_add_u32 s4, s4, G at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s5, s5, G at gotpcrel32@hi+12
@@ -47,12 +47,10 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: s_mov_b32 s52, s15
; CHECK-NEXT: s_mov_b64 s[36:37], s[10:11]
; CHECK-NEXT: v_mov_b32_e32 v40, v0
-; CHECK-NEXT: v_mov_b32_e32 v60, s66
-; CHECK-NEXT: v_mov_b32_e32 v61, s67
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[62:63]
+; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[60:61]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
-; CHECK-NEXT: flat_load_dwordx2 a[32:33], v[56:57]
+; CHECK-NEXT: flat_load_dwordx2 v[62:63], v[56:57]
; CHECK-NEXT: v_mov_b32_e32 v58, 0
; CHECK-NEXT: v_mov_b32_e32 v59, 0x3ff00000
; CHECK-NEXT: s_mov_b64 s[4:5], s[48:49]
@@ -64,18 +62,18 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: s_mov_b32 s14, s33
; CHECK-NEXT: v_mov_b32_e32 v31, v40
; CHECK-NEXT: flat_store_dwordx2 v[44:45], v[58:59]
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[62:63]
+; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[60:61]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: ; implicit-def: $sgpr15
; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[46:47] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s67
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[66:67], s[66:67] op_sel:[0,1]
+; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[62:63]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[0:1]
; CHECK-NEXT: v_mov_b32_e32 v0, s68
; CHECK-NEXT: v_cmp_lt_i32_e32 vcc, 0, v42
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], a[32:33]
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[60:61]
; CHECK-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_store_dword v58, v0, s[0:3], 0 offen
; CHECK-NEXT: ; implicit-def: $vgpr4
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmaximum.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmaximum.ll
index 3ac13782301dc..05a46a588de11 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmaximum.ll
@@ -867,50 +867,50 @@ define half @test_vector_reduce_fmaximum_v16half(<16 x half> %v) {
; GFX9-LABEL: test_vector_reduce_fmaximum_v16half:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_max_f16 v8, v1, v5
+; GFX9-NEXT: v_pk_max_f16 v8, v3, v7
; GFX9-NEXT: v_mov_b32_e32 v9, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v5
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v3, v7
; GFX9-NEXT: s_mov_b32 s0, 0x5040100
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v10, v9, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc
-; GFX9-NEXT: v_pk_max_f16 v8, v0, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v4
-; GFX9-NEXT: v_perm_b32 v5, v1, v10, s0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v8, vcc
+; GFX9-NEXT: v_pk_max_f16 v8, v1, v5
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v5
+; GFX9-NEXT: v_perm_b32 v7, v3, v10, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v11, v9, v8, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_pk_max_f16 v8, v3, v7
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v3, v7
-; GFX9-NEXT: v_perm_b32 v4, v0, v11, s0
+; GFX9-NEXT: v_cndmask_b32_sdwa v1, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_pk_max_f16 v8, v2, v6
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v2, v6
+; GFX9-NEXT: v_perm_b32 v5, v1, v11, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v12, v9, v8, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_pk_max_f16 v7, v2, v6
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v2, v6
-; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v7, vcc
; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v9, v7, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_perm_b32 v6, v2, v8, s0
+; GFX9-NEXT: v_cndmask_b32_sdwa v2, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_pk_max_f16 v8, v0, v4
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v4
+; GFX9-NEXT: v_perm_b32 v6, v2, v12, s0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v9, v8, vcc
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: s_nop 1
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_perm_b32 v4, v0, v13, s0
; GFX9-NEXT: v_pk_max_f16 v4, v4, v6
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v11, v8
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v13, v12
; GFX9-NEXT: s_nop 1
; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v4, vcc
; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
; GFX9-NEXT: s_nop 1
; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_perm_b32 v4, v3, v12, s0
-; GFX9-NEXT: v_pk_max_f16 v4, v5, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v10, v12
+; GFX9-NEXT: v_pk_max_f16 v4, v5, v7
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v11, v10
; GFX9-NEXT: v_perm_b32 v2, v0, v6, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v4, vcc
@@ -934,42 +934,41 @@ define half @test_vector_reduce_fmaximum_v16half(<16 x half> %v) {
; GFX10-LABEL: test_vector_reduce_fmaximum_v16half:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_pk_max_f16 v8, v1, v5
+; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_max_f16 v8, v3, v7
; GFX10-NEXT: v_mov_b32_e32 v9, 0x7e00
-; GFX10-NEXT: v_pk_max_f16 v10, v0, v4
-; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_sdwa s5, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_max_f16 v10, v1, v5
; GFX10-NEXT: v_pk_max_f16 v12, v2, v6
; GFX10-NEXT: v_cndmask_b32_sdwa v11, v9, v8, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v10, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v10, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_pk_max_f16 v10, v3, v7
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v2, v6
; GFX10-NEXT: v_cndmask_b32_sdwa v13, v9, v10, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v12, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_max_f16 v6, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v12, s4
+; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_cndmask_b32_sdwa v12, v9, v12, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v9, v12, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX10-NEXT: v_perm_b32 v5, v4, v0, 0x5040100
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, v0, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX10-NEXT: v_perm_b32 v8, v6, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v7, v11, v1, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v10, vcc_lo
-; GFX10-NEXT: v_pk_max_f16 v5, v5, v8
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v6
-; GFX10-NEXT: v_perm_b32 v8, v13, v3, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0x7e00, v5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, v11, v13
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_perm_b32 v6, v12, v2, 0x5040100
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v0, v2
+; GFX10-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX10-NEXT: v_pk_max_f16 v6, v7, v6
+; GFX10-NEXT: v_perm_b32 v5, v11, v3, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v10, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0x7e00, v6, s4
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v13, v11
+; GFX10-NEXT: v_perm_b32 v7, v13, v1, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT: v_pk_max_f16 v2, v7, v8
+; GFX10-NEXT: v_pk_max_f16 v2, v7, v5
; GFX10-NEXT: v_perm_b32 v3, v4, v0, 0x5040100
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
@@ -988,52 +987,52 @@ define half @test_vector_reduce_fmaximum_v16half(<16 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmaximum_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v11, v0, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v8, v1, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v10.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, 0x7e00, v8.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, 0x7e00, v11.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, 0x7e00, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v0, v3, v7
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v10, v2, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v7
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v2.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v9.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v10
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s3, v12.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, 0x7e00, v10.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.h, 0x7e00, v2.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, 0x7e00, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v0.l, s2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v3.l, s3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v1, v5, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v5.h, v6.h
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v2, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v4.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v8, v3, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v9.l, 0x7e00, v8.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v1.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v1, v1, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v11.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v2.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v2, v2, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v10, v0, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s3, v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s4, v0.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s5, v8.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0x7e00, v2.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x7e00, v5.l, s3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, 0x7e00, v10.l, s4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, 0x7e00, v0.l, s5
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v9.h, 0x7e00, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v4.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v1, v5, v2
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v5.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v0.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v3, v0, v9
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v0.h, v9.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v1.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x7e00, v2.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v3.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x7e00, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v4.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x7e00, v5.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x7e00, v2.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_pk_max_f16 v2, v0, v1
@@ -1051,68 +1050,67 @@ define half @test_vector_reduce_fmaximum_v16half(<16 x half> %v) {
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fmaximum_v16half:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v8, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v10, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v4
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v10
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v5, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v8, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v10, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v13, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v13, v12
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v6, 0x7e00, v14, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v8, v3, v7
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v5, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v14, v0, v4
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v8
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v7, v1, v5
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v10, v9
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v9, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 16, v14
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v8, 0x7e00, v11, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v13, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v9, v8, v3, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v14, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v16, v15
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v17, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v7, vcc_lo
; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v10, v1, v9, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v5, 0x7e00, v15, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v8, v6, v2, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v6, v2, v13, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v5, 0x7e00, v12, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v13
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v11, v5, v3, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v7, v7, v8
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v10, v5, v1, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v6, v7, v6
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v8, v10, v11
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v6
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v7, v10, v9
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v10, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v9, v3
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v2, v4, v0, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v3
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v7, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5, v8
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v6, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v4, v2, v0, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v3
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v5, v1, v3, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v5, v3, v1, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v2, v2, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v1
+; GFX11-SDAG-FAKE16-NEXT: v_pk_max_f16 v4, v4, v5
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v3
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v5, vcc_lo
; GFX11-SDAG-FAKE16-NEXT: v_max_f16_e32 v2, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
index 1ad17d8b89b5a..bc941091a170e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
@@ -1047,50 +1047,50 @@ define half @test_vector_reduce_fminimum_v16half(<16 x half> %v) {
; GFX9-LABEL: test_vector_reduce_fminimum_v16half:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_min_f16 v8, v1, v5
+; GFX9-NEXT: v_pk_min_f16 v8, v3, v7
; GFX9-NEXT: v_mov_b32_e32 v9, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v5
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v3, v7
; GFX9-NEXT: s_mov_b32 s0, 0x5040100
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v10, v9, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc
-; GFX9-NEXT: v_pk_min_f16 v8, v0, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v4
-; GFX9-NEXT: v_perm_b32 v5, v1, v10, s0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v8, vcc
+; GFX9-NEXT: v_pk_min_f16 v8, v1, v5
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v5
+; GFX9-NEXT: v_perm_b32 v7, v3, v10, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v11, v9, v8, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_pk_min_f16 v8, v3, v7
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v3, v7
-; GFX9-NEXT: v_perm_b32 v4, v0, v11, s0
+; GFX9-NEXT: v_cndmask_b32_sdwa v1, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_pk_min_f16 v8, v2, v6
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v2, v6
+; GFX9-NEXT: v_perm_b32 v5, v1, v11, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v12, v9, v8, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_pk_min_f16 v7, v2, v6
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v2, v6
-; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v7, vcc
; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v9, v7, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_perm_b32 v6, v2, v8, s0
+; GFX9-NEXT: v_cndmask_b32_sdwa v2, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_pk_min_f16 v8, v0, v4
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v4
+; GFX9-NEXT: v_perm_b32 v6, v2, v12, s0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v9, v8, vcc
+; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT: s_nop 1
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v8, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_perm_b32 v4, v0, v13, s0
; GFX9-NEXT: v_pk_min_f16 v4, v4, v6
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v11, v8
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v13, v12
; GFX9-NEXT: s_nop 1
; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v4, vcc
; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
; GFX9-NEXT: s_nop 1
; GFX9-NEXT: v_cndmask_b32_sdwa v0, v9, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_perm_b32 v4, v3, v12, s0
-; GFX9-NEXT: v_pk_min_f16 v4, v5, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v10, v12
+; GFX9-NEXT: v_pk_min_f16 v4, v5, v7
+; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v11, v10
; GFX9-NEXT: v_perm_b32 v2, v0, v6, s0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v4, vcc
@@ -1114,42 +1114,41 @@ define half @test_vector_reduce_fminimum_v16half(<16 x half> %v) {
; GFX10-LABEL: test_vector_reduce_fminimum_v16half:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_pk_min_f16 v8, v1, v5
+; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_min_f16 v8, v3, v7
; GFX10-NEXT: v_mov_b32_e32 v9, 0x7e00
-; GFX10-NEXT: v_pk_min_f16 v10, v0, v4
-; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_sdwa s5, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_min_f16 v10, v1, v5
; GFX10-NEXT: v_pk_min_f16 v12, v2, v6
; GFX10-NEXT: v_cndmask_b32_sdwa v11, v9, v8, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v10, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v10, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_pk_min_f16 v10, v3, v7
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v2, v6
; GFX10-NEXT: v_cndmask_b32_sdwa v13, v9, v10, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v12, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_sdwa vcc_lo, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_pk_min_f16 v6, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v12, s4
+; GFX10-NEXT: v_cmp_o_f16_sdwa s4, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT: v_cndmask_b32_sdwa v12, v9, v12, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v9, v12, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX10-NEXT: v_perm_b32 v5, v4, v0, 0x5040100
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, v0, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX10-NEXT: v_perm_b32 v8, v6, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v7, v11, v1, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v10, vcc_lo
-; GFX10-NEXT: v_pk_min_f16 v5, v5, v8
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v6
-; GFX10-NEXT: v_perm_b32 v8, v13, v3, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0x7e00, v5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, v11, v13
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_perm_b32 v6, v12, v2, 0x5040100
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v0, v2
+; GFX10-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX10-NEXT: v_pk_min_f16 v6, v7, v6
+; GFX10-NEXT: v_perm_b32 v5, v11, v3, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v10, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0x7e00, v6, s4
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, v13, v11
+; GFX10-NEXT: v_perm_b32 v7, v13, v1, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_sdwa v4, v9, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT: v_pk_min_f16 v2, v7, v8
+; GFX10-NEXT: v_pk_min_f16 v2, v7, v5
; GFX10-NEXT: v_perm_b32 v3, v4, v0, 0x5040100
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
; GFX10-NEXT: s_mov_b32 vcc_lo, s4
@@ -1168,52 +1167,52 @@ define half @test_vector_reduce_fminimum_v16half(<16 x half> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fminimum_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v11, v0, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v8, v1, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v10.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, 0x7e00, v8.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, 0x7e00, v11.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, 0x7e00, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v0, v3, v7
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v10, v2, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v7
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v2.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v9.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v10
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s3, v12.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, 0x7e00, v10.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.h, 0x7e00, v2.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, 0x7e00, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v0.l, s2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v3.l, s3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v1, v5, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v5.h, v6.h
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v2, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v4.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v8, v3, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v9.l, 0x7e00, v8.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v1.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v1, v1, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v11.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v2.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v2, v2, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v10, v0, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s3, v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s4, v0.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s5, v8.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0x7e00, v2.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x7e00, v5.l, s3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, 0x7e00, v10.l, s4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, 0x7e00, v0.l, s5
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v9.h, 0x7e00, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v4.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v1, v5, v2
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v5.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s1, v0.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v3, v0, v9
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e64 s2, v0.h, v9.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v1.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x7e00, v2.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v3.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x7e00, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x7e00, v4.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x7e00, v5.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x7e00, v2.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_pk_min_f16 v2, v0, v1
@@ -1231,68 +1230,67 @@ define half @test_vector_reduce_fminimum_v16half(<16 x half> %v) {
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fminimum_v16half:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v8, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v10, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v4
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v10
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v5, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v8, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v10, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v13, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v13, v12
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v6, 0x7e00, v14, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v8, v3, v7
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v5, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v14, v0, v4
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v8
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v7, v1, v5
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v10, v9
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v9, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v17, 16, v14
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v8, 0x7e00, v11, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v13, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v6
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v4
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v9, v8, v3, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v14, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v16, v15
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v17, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v7, vcc_lo
; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v10, v1, v9, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v5, 0x7e00, v15, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v8, v6, v2, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v6, v2, v13, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v5, 0x7e00, v12, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v7, v4, v0, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v13
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v11, v5, v3, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v7, v7, v8
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v10, v5, v1, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v6, v7, v6
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v8, v10, v11
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v6
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v7, v10, v9
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v4, 0x7e00, v10, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v9, v3
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v2, v4, v0, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v3
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v7, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v5, v8
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v6, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v4, v2, v0, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v3
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v5, v1, v3, 0x5040100
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v5, v3, v1, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v2, v2, v5
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v4, v1
+; GFX11-SDAG-FAKE16-NEXT: v_pk_min_f16 v4, v4, v5
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v2, v3
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v5, vcc_lo
; GFX11-SDAG-FAKE16-NEXT: v_min_f16_e32 v2, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index a0910fa124f67..2b59ae12d1b89 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -5318,46 +5318,46 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
;
; GFX942-LABEL: fma_shuffle_v2bf16:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v6, 3, v0
; GFX942-NEXT: s_movk_i32 s2, 0x7fff
; GFX942-NEXT: s_mov_b32 s3, 0x7060302
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[8:9]
; GFX942-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[8:9]
; GFX942-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11]
; GFX942-NEXT: s_waitcnt vmcnt(2)
-; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v8, 16, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX942-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
+; GFX942-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v12, 16, v1
; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX942-NEXT: v_fmac_f32_e32 v8, v7, v9
-; GFX942-NEXT: v_fmac_f32_e32 v2, v7, v4
-; GFX942-NEXT: v_fmac_f32_e32 v3, v11, v4
-; GFX942-NEXT: v_bfe_u32 v4, v8, 16, 1
-; GFX942-NEXT: v_fmac_f32_e32 v12, v11, v9
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v8
+; GFX942-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX942-NEXT: v_fmac_f32_e32 v2, v8, v4
+; GFX942-NEXT: v_fmac_f32_e32 v3, v12, v4
+; GFX942-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX942-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v7
; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX942-NEXT: v_add3_u32 v4, v4, v8, s2
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
-; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX942-NEXT: v_bfe_u32 v13, v12, 16, 1
+; GFX942-NEXT: v_add3_u32 v4, v4, v7, s2
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX942-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX942-NEXT: v_bfe_u32 v13, v11, 16, 1
; GFX942-NEXT: v_add3_u32 v9, v9, v2, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_or_b32_e32 v14, 0x400000, v12
+; GFX942-NEXT: v_or_b32_e32 v14, 0x400000, v11
; GFX942-NEXT: v_bfe_u32 v15, v3, 16, 1
-; GFX942-NEXT: v_add3_u32 v13, v13, v12, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX942-NEXT: v_add3_u32 v13, v13, v11, s2
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: v_or_b32_e32 v16, 0x400000, v3
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
index 735b8e0f74089..f4ef18b2eb1f0 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
@@ -389,14 +389,14 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: v_writelane_b32 v93, s30, 12
; CHECK-NEXT: v_writelane_b32 v93, s31, 13
; CHECK-NEXT: v_dual_mov_b32 v92, v31 :: v_dual_mov_b32 v1, 0
-; CHECK-NEXT: v_dual_mov_b32 v0, 0x60 :: v_dual_mov_b32 v3, 0
-; CHECK-NEXT: v_dual_mov_b32 v2, 0x50 :: v_dual_mov_b32 v5, 0
-; CHECK-NEXT: v_dual_mov_b32 v4, 64 :: v_dual_mov_b32 v7, 0
+; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 0
+; CHECK-NEXT: v_dual_mov_b32 v2, 16 :: v_dual_mov_b32 v5, 0
+; CHECK-NEXT: v_dual_mov_b32 v4, 32 :: v_dual_mov_b32 v7, 0
; CHECK-NEXT: v_mov_b32_e32 v6, 48
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: global_load_b128 v[56:59], v[0:1], off
; CHECK-NEXT: global_load_b128 v[104:107], v[2:3], off
-; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 0
+; CHECK-NEXT: v_dual_mov_b32 v0, 0x60 :: v_dual_mov_b32 v3, 0
; CHECK-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0x70
; CHECK-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, 0x80
; CHECK-NEXT: s_clause 0x1
@@ -409,8 +409,8 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: global_load_b128 v[14:17], v[8:9], off
; CHECK-NEXT: global_load_b128 v[18:21], v[8:9], off offset:16
; CHECK-NEXT: global_load_b128 v[22:25], v[8:9], off offset:32
-; CHECK-NEXT: v_dual_mov_b32 v4, 32 :: v_dual_mov_b32 v7, 0
-; CHECK-NEXT: v_mov_b32_e32 v6, 16
+; CHECK-NEXT: v_dual_mov_b32 v4, 64 :: v_dual_mov_b32 v7, 0
+; CHECK-NEXT: v_mov_b32_e32 v6, 0x50
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: global_load_b128 v[76:79], v[4:5], off
; CHECK-NEXT: global_load_b128 v[88:91], v[6:7], off
@@ -508,19 +508,19 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: scratch_store_b128 off, v[11:14], s32 offset:96
; CHECK-NEXT: scratch_store_b128 off, v[7:10], s32 offset:80
; CHECK-NEXT: scratch_store_b128 off, v[3:6], s32 offset:64
-; CHECK-NEXT: v_dual_mov_b32 v4, v88 :: v_dual_mov_b32 v5, v89
-; CHECK-NEXT: v_dual_mov_b32 v6, v90 :: v_dual_mov_b32 v7, v91
-; CHECK-NEXT: v_dual_mov_b32 v8, v76 :: v_dual_mov_b32 v9, v77
-; CHECK-NEXT: v_dual_mov_b32 v10, v78 :: v_dual_mov_b32 v11, v79
+; CHECK-NEXT: v_dual_mov_b32 v4, v104 :: v_dual_mov_b32 v5, v105
+; CHECK-NEXT: v_dual_mov_b32 v6, v106 :: v_dual_mov_b32 v7, v107
+; CHECK-NEXT: v_dual_mov_b32 v8, v108 :: v_dual_mov_b32 v9, v109
+; CHECK-NEXT: v_dual_mov_b32 v10, v110 :: v_dual_mov_b32 v11, v111
; CHECK-NEXT: v_dual_mov_b32 v12, v60 :: v_dual_mov_b32 v13, v61
; CHECK-NEXT: v_dual_mov_b32 v14, v62 :: v_dual_mov_b32 v15, v63
-; CHECK-NEXT: v_dual_mov_b32 v16, v108 :: v_dual_mov_b32 v17, v109
-; CHECK-NEXT: v_mov_b32_e32 v18, v110
+; CHECK-NEXT: v_dual_mov_b32 v16, v76 :: v_dual_mov_b32 v17, v77
+; CHECK-NEXT: v_mov_b32_e32 v18, v78
; CHECK-NEXT: s_wait_loadcnt 0x1
; CHECK-NEXT: v_dual_mov_b32 v44, v0 :: v_dual_mov_b32 v45, v1
; CHECK-NEXT: v_mov_b32_e32 v46, v2
-; CHECK-NEXT: v_dual_mov_b32 v0, v72 :: v_dual_mov_b32 v1, v73
-; CHECK-NEXT: v_dual_mov_b32 v2, v74 :: v_dual_mov_b32 v3, v75
+; CHECK-NEXT: v_dual_mov_b32 v0, v56 :: v_dual_mov_b32 v1, v57
+; CHECK-NEXT: v_dual_mov_b32 v2, v58 :: v_dual_mov_b32 v3, v59
; CHECK-NEXT: v_dual_mov_b32 v43, v34 :: v_dual_mov_b32 v42, v33
; CHECK-NEXT: v_dual_mov_b32 v41, v32 :: v_dual_mov_b32 v40, v31
; CHECK-NEXT: v_dual_mov_b32 v39, v30 :: v_dual_mov_b32 v38, v29
@@ -538,11 +538,11 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: scratch_store_b128 off, v[35:38], s32 offset:16
; CHECK-NEXT: scratch_store_b128 off, v[31:34], s32
; CHECK-NEXT: v_mov_b32_e32 v31, v92
-; CHECK-NEXT: v_dual_mov_b32 v19, v111 :: v_dual_mov_b32 v20, v104
-; CHECK-NEXT: v_dual_mov_b32 v21, v105 :: v_dual_mov_b32 v22, v106
-; CHECK-NEXT: v_dual_mov_b32 v23, v107 :: v_dual_mov_b32 v24, v56
-; CHECK-NEXT: v_dual_mov_b32 v25, v57 :: v_dual_mov_b32 v26, v58
-; CHECK-NEXT: v_mov_b32_e32 v27, v59
+; CHECK-NEXT: v_dual_mov_b32 v19, v79 :: v_dual_mov_b32 v20, v88
+; CHECK-NEXT: v_dual_mov_b32 v21, v89 :: v_dual_mov_b32 v22, v90
+; CHECK-NEXT: v_dual_mov_b32 v23, v91 :: v_dual_mov_b32 v24, v72
+; CHECK-NEXT: v_dual_mov_b32 v25, v73 :: v_dual_mov_b32 v26, v74
+; CHECK-NEXT: v_mov_b32_e32 v27, v75
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index 61ad66d2b3b33..0df8598c9e815 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -218,21 +218,21 @@ define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1
; GFX942-NEXT: v_lshlrev_b32_e32 v1, 5, v0
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB5_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 5, v0
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3]
; GFX942-NEXT: .LBB5_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:16
+; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:16
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -260,69 +260,77 @@ define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:240
-; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:224
-; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[0:1] offset:208
-; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[0:1] offset:192
-; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[0:1] offset:176
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:160
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:144
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:128
-; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1] offset:112
-; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[0:1] offset:96
-; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[0:1] offset:80
-; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[0:1] offset:64
-; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[0:1] offset:48
-; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:32
-; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:240
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:224
+; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:208
+; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[0:1] offset:192
+; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[0:1] offset:176
+; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[0:1] offset:160
+; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:144
+; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:128
+; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1] offset:112
+; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:96
+; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:80
+; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[0:1] offset:64
+; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[0:1] offset:48
+; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[0:1] offset:32
+; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB6_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX942-NEXT: global_load_dwordx4 v[30:33], v0, s[2:3] offset:240
-; GFX942-NEXT: global_load_dwordx4 v[26:29], v0, s[2:3] offset:224
-; GFX942-NEXT: global_load_dwordx4 v[22:25], v0, s[2:3] offset:208
-; GFX942-NEXT: global_load_dwordx4 v[18:21], v0, s[2:3] offset:192
-; GFX942-NEXT: global_load_dwordx4 v[14:17], v0, s[2:3] offset:176
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v0, s[2:3] offset:160
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:144
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:128
-; GFX942-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3] offset:112
-; GFX942-NEXT: global_load_dwordx4 v[58:61], v0, s[2:3] offset:96
-; GFX942-NEXT: global_load_dwordx4 v[54:57], v0, s[2:3] offset:80
-; GFX942-NEXT: global_load_dwordx4 v[50:53], v0, s[2:3] offset:64
-; GFX942-NEXT: global_load_dwordx4 v[46:49], v0, s[2:3] offset:48
-; GFX942-NEXT: global_load_dwordx4 v[42:45], v0, s[2:3] offset:32
-; GFX942-NEXT: global_load_dwordx4 v[38:41], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[34:37], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:240
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:224
+; GFX942-NEXT: global_load_dwordx4 v[10:13], v0, s[2:3] offset:208
+; GFX942-NEXT: global_load_dwordx4 v[14:17], v0, s[2:3] offset:192
+; GFX942-NEXT: global_load_dwordx4 v[18:21], v0, s[2:3] offset:176
+; GFX942-NEXT: global_load_dwordx4 v[22:25], v0, s[2:3] offset:160
+; GFX942-NEXT: global_load_dwordx4 v[26:29], v0, s[2:3] offset:144
+; GFX942-NEXT: global_load_dwordx4 v[30:33], v0, s[2:3] offset:128
+; GFX942-NEXT: global_load_dwordx4 v[34:37], v0, s[2:3] offset:112
+; GFX942-NEXT: global_load_dwordx4 v[38:41], v0, s[2:3] offset:96
+; GFX942-NEXT: global_load_dwordx4 v[42:45], v0, s[2:3] offset:80
+; GFX942-NEXT: global_load_dwordx4 v[46:49], v0, s[2:3] offset:64
+; GFX942-NEXT: global_load_dwordx4 v[50:53], v0, s[2:3] offset:48
+; GFX942-NEXT: global_load_dwordx4 v[54:57], v0, s[2:3] offset:32
+; GFX942-NEXT: global_load_dwordx4 v[58:61], v0, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3]
; GFX942-NEXT: .LBB6_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] offset:112
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[58:61], s[6:7] offset:96
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[54:57], s[6:7] offset:80
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[50:53], s[6:7] offset:64
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[46:49], s[6:7] offset:48
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[42:45], s[6:7] offset:32
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[38:41], s[6:7] offset:16
-; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[34:37], s[6:7]
-; GFX942-NEXT: global_store_dwordx4 v1, v[30:33], s[6:7] offset:240
-; GFX942-NEXT: global_store_dwordx4 v1, v[26:29], s[6:7] offset:224
-; GFX942-NEXT: global_store_dwordx4 v1, v[22:25], s[6:7] offset:208
-; GFX942-NEXT: global_store_dwordx4 v1, v[18:21], s[6:7] offset:192
-; GFX942-NEXT: global_store_dwordx4 v1, v[14:17], s[6:7] offset:176
-; GFX942-NEXT: global_store_dwordx4 v1, v[10:13], s[6:7] offset:160
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:144
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:128
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:240
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:224
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[10:13], s[6:7] offset:208
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[14:17], s[6:7] offset:192
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[18:21], s[6:7] offset:176
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[22:25], s[6:7] offset:160
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[26:29], s[6:7] offset:144
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[30:33], s[6:7] offset:128
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[34:37], s[6:7] offset:112
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[38:41], s[6:7] offset:96
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[42:45], s[6:7] offset:80
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[46:49], s[6:7] offset:64
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[50:53], s[6:7] offset:48
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[54:57], s[6:7] offset:32
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, v[58:61], s[6:7] offset:16
+; GFX942-NEXT: s_waitcnt vmcnt(15)
+; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index a3107fb580395..ca5c347fcef6c 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -135,7 +135,12 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_absC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GCN-LABEL: test_wmma_f16_16x16x16_f16_absC:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_hi:[0,0,1]
+; GCN-NEXT: v_and_b32_e32 v11, 0x7fff7fff, v11
+; GCN-NEXT: v_and_b32_e32 v10, 0x7fff7fff, v10
+; GCN-NEXT: v_and_b32_e32 v9, 0x7fff7fff, v9
+; GCN-NEXT: v_and_b32_e32 v8, 0x7fff7fff, v8
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11]
; GCN-NEXT: global_store_b128 v[12:13], v[8:11], off
; GCN-NEXT: s_endpgm
bb:
@@ -342,7 +347,12 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negabsC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GCN-LABEL: test_wmma_f16_16x16x16_f16_negabsC:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT: v_and_b32_e32 v11, 0x7fff7fff, v11
+; GCN-NEXT: v_and_b32_e32 v10, 0x7fff7fff, v10
+; GCN-NEXT: v_and_b32_e32 v9, 0x7fff7fff, v9
+; GCN-NEXT: v_and_b32_e32 v8, 0x7fff7fff, v8
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[0,0,1]
; GCN-NEXT: global_store_b128 v[12:13], v[8:11], off
; GCN-NEXT: s_endpgm
bb:
>From 641575c12f8e2351daeefe2045f8cd9981baa285 Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Thu, 6 Aug 2026 09:39:50 +0800
Subject: [PATCH 2/4] fix tests
---
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 4 ++--
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index 043591f999536..925480a28f798 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -105,7 +105,7 @@ define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
; GFX1251-LABEL: fadd_v4_vs:
; GFX1251: ; %bb.0:
-; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
@@ -979,7 +979,7 @@ define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
; GFX1251-LABEL: fmul_v4_vs:
; GFX1251: ; %bb.0:
-; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index c7574b81cf0e3..0960304aea383 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -105,7 +105,7 @@ define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
define amdgpu_kernel void @add_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
; GFX1251-LABEL: add_v4_vs:
; GFX1251: ; %bb.0:
-; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
@@ -731,7 +731,7 @@ define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
define amdgpu_kernel void @sub_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
; GFX1251-LABEL: sub_v4_vs:
; GFX1251: ; %bb.0:
-; GFX1251-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1251-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
>From 86e729f38b68a801d7122439d3e3dbf9319817fe Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Thu, 6 Aug 2026 14:05:36 +0800
Subject: [PATCH 3/4] fix DAG mismatch
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 8 +-
llvm/test/CodeGen/AMDGPU/sra.ll | 20 ++--
llvm/test/CodeGen/ARM/pr196779.ll | 92 +++++++++----------
llvm/test/CodeGen/X86/var-permute-128.ll | 24 ++---
4 files changed, 69 insertions(+), 75 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 5261b3e9f40fe..84968e57e9b06 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -28274,9 +28274,11 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
// an illegal source vector would be promoted straight back into the
// extract_subvector we started from.
if (NewExtEC.isScalar() &&
- TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT) &&
- (!LegalOperations ||
- TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT) &&
+ !(TLI.getTypeAction(*DAG.getContext(), NVT) ==
+ TargetLoweringBase::TypePromoteInteger &&
+ TLI.getTypeAction(*DAG.getContext(), SrcVT) !=
+ TargetLoweringBase::TypePromoteInteger)) {
SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
SDValue NewExtract =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT,
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index 063af59339ba3..55a3f59ef6f40 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -205,7 +205,7 @@ define amdgpu_kernel void @ashr_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; EG: ; %bb.0:
; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
; EG-NEXT: TEX 0 @6
-; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[]
; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T6.X, T7.X, 1
; EG-NEXT: CF_END
; EG-NEXT: PAD
@@ -214,19 +214,17 @@ define amdgpu_kernel void @ashr_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; EG-NEXT: ALU clause starting at 8:
; EG-NEXT: MOV * T6.X, KC0[2].Z,
; EG-NEXT: ALU clause starting at 9:
-; EG-NEXT: LSHR * T0.W, T6.X, literal.x,
-; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
-; EG-NEXT: LSHR T0.Z, T6.Y, literal.x,
-; EG-NEXT: BFE_INT T0.W, T6.X, 0.0, literal.x,
-; EG-NEXT: AND_INT * T1.W, T6.Y, literal.y,
+; EG-NEXT: BFE_INT T0.Y, T6.X, 0.0, literal.x,
+; EG-NEXT: AND_INT T0.Z, T6.Y, literal.y,
+; EG-NEXT: ASHR T0.W, T6.X, literal.x,
+; EG-NEXT: LSHR * T1.W, T6.Y, literal.x,
; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41)
; EG-NEXT: ASHR T0.W, PV.W, PS,
; EG-NEXT: ASHR * T1.W, PV.Y, PV.Z,
-; EG-NEXT: LSHL T1.W, PS, literal.x,
-; EG-NEXT: AND_INT * T0.W, PV.W, literal.y,
-; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41)
-; EG-NEXT: OR_INT T6.X, PS, PV.W,
+; EG-NEXT: AND_INT T1.W, PS, literal.x,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 16(2.242078e-44)
+; EG-NEXT: OR_INT T6.X, PV.W, PS,
; EG-NEXT: LSHR * T7.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%b_ptr = getelementptr <2 x i16>, ptr addrspace(1) %in, i16 1
diff --git a/llvm/test/CodeGen/ARM/pr196779.ll b/llvm/test/CodeGen/ARM/pr196779.ll
index 536c87854ade4..db2aab9c5945f 100644
--- a/llvm/test/CodeGen/ARM/pr196779.ll
+++ b/llvm/test/CodeGen/ARM/pr196779.ll
@@ -8,82 +8,76 @@ define void @repro(ptr %out, ptr %base) {
; CHECK-NEXT: sub sp, sp, #4
; CHECK-NEXT: mov r0, #1
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: bne .LBB0_3
+; CHECK-NEXT: bne .LBB0_5
; CHECK-NEXT: @ %bb.1: @ %cond.load
; CHECK-NEXT: vmov.i32 d16, #0x0
-; CHECK-NEXT: vldr d18, [r0]
+; CHECK-NEXT: vldr d17, [r0]
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: beq .LBB0_4
+; CHECK-NEXT: beq .LBB0_6
; CHECK-NEXT: .LBB0_2:
-; CHECK-NEXT: vorr d20, d16, d16
-; CHECK-NEXT: b .LBB0_5
+; CHECK-NEXT: vorr d18, d16, d16
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: beq .LBB0_7
; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT: vorr d19, d16, d16
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: beq .LBB0_8
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: vorr d20, d16, d16
+; CHECK-NEXT: b .LBB0_9
+; CHECK-NEXT: .LBB0_5:
; CHECK-NEXT: mov r2, #0
-; CHECK-NEXT: vmov d18, r2, r2
-; CHECK-NEXT: vorr d16, d18, d18
+; CHECK-NEXT: vmov d17, r2, r2
+; CHECK-NEXT: vorr d16, d17, d17
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: bne .LBB0_2
-; CHECK-NEXT: .LBB0_4: @ %cond.load1
-; CHECK-NEXT: ldm r0, {r0, r3}
-; CHECK-NEXT: vmov d20, r0, r3
-; CHECK-NEXT: .LBB0_5: @ %else2
-; CHECK-NEXT: mov r0, #1
+; CHECK-NEXT: .LBB0_6: @ %cond.load1
+; CHECK-NEXT: vldr d18, [r0]
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: bne .LBB0_8
-; CHECK-NEXT: @ %bb.6: @ %cond.load4
-; CHECK-NEXT: ldrd r2, r3, [r1]
-; CHECK-NEXT: vmov d22, r2, r3
-; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: beq .LBB0_9
-; CHECK-NEXT: .LBB0_7:
-; CHECK-NEXT: vorr d21, d16, d16
-; CHECK-NEXT: b .LBB0_10
-; CHECK-NEXT: .LBB0_8:
-; CHECK-NEXT: vorr d22, d16, d16
-; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: bne .LBB0_7
-; CHECK-NEXT: .LBB0_9: @ %cond.load7
-; CHECK-NEXT: ldrd r0, r1, [r0]
-; CHECK-NEXT: vmov d21, r0, r1
-; CHECK-NEXT: .LBB0_10: @ %else8
-; CHECK-NEXT: mov r0, #1
+; CHECK-NEXT: bne .LBB0_3
+; CHECK-NEXT: .LBB0_7: @ %cond.load4
+; CHECK-NEXT: vldr d19, [r1]
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: bne .LBB0_4
+; CHECK-NEXT: .LBB0_8: @ %cond.load7
+; CHECK-NEXT: vldr d20, [r0]
+; CHECK-NEXT: .LBB0_9: @ %else8
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: ldrdeq r2, r3, [r0]
-; CHECK-NEXT: vmoveq d16, r2, r3
+; CHECK-NEXT: vldreq d16, [r0]
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: beq .LBB0_15
+; CHECK-NEXT: @ %bb.10: @ %else14
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: beq .LBB0_16
-; CHECK-NEXT: @ %bb.11: @ %else14
+; CHECK-NEXT: .LBB0_11: @ %else16
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: beq .LBB0_17
-; CHECK-NEXT: .LBB0_12: @ %else16
+; CHECK-NEXT: .LBB0_12: @ %else18
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: beq .LBB0_18
-; CHECK-NEXT: .LBB0_13: @ %else18
+; CHECK-NEXT: .LBB0_13: @ %else20
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: beq .LBB0_19
-; CHECK-NEXT: .LBB0_14: @ %else20
-; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: beq .LBB0_20
-; CHECK-NEXT: .LBB0_15: @ %else22
+; CHECK-NEXT: .LBB0_14: @ %else22
; CHECK-NEXT: add sp, sp, #4
; CHECK-NEXT: bx lr
-; CHECK-NEXT: .LBB0_16: @ %cond.store
+; CHECK-NEXT: .LBB0_15: @ %cond.store
+; CHECK-NEXT: vst1.8 {d17}, [r0]
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: bne .LBB0_11
+; CHECK-NEXT: .LBB0_16: @ %cond.store15
; CHECK-NEXT: vst1.8 {d18}, [r0]
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: bne .LBB0_12
-; CHECK-NEXT: .LBB0_17: @ %cond.store15
-; CHECK-NEXT: vst1.8 {d20}, [r0]
+; CHECK-NEXT: .LBB0_17: @ %cond.store17
+; CHECK-NEXT: vst1.8 {d19}, [r0]
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: bne .LBB0_13
-; CHECK-NEXT: .LBB0_18: @ %cond.store17
-; CHECK-NEXT: vst1.8 {d22}, [r0]
+; CHECK-NEXT: .LBB0_18: @ %cond.store19
+; CHECK-NEXT: vst1.8 {d20}, [r0]
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: bne .LBB0_14
-; CHECK-NEXT: .LBB0_19: @ %cond.store19
-; CHECK-NEXT: vst1.8 {d21}, [r0]
-; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: bne .LBB0_15
-; CHECK-NEXT: .LBB0_20: @ %cond.store21
+; CHECK-NEXT: .LBB0_19: @ %cond.store21
; CHECK-NEXT: vst1.8 {d16}, [r0]
; CHECK-NEXT: add sp, sp, #4
; CHECK-NEXT: bx lr
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index dbfad9562d65a..84e79f99e7d00 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -2027,13 +2027,13 @@ define void @indices_convert() {
; SSE3-LABEL: indices_convert:
; SSE3: # %bb.0: # %bb
; SSE3-NEXT: movaps (%rax), %xmm0
+; SSE3-NEXT: movl (%rax), %eax
; SSE3-NEXT: movaps %xmm0, -24(%rsp)
; SSE3-NEXT: movaps %xmm0, -40(%rsp)
-; SSE3-NEXT: movl (%rax), %eax
-; SSE3-NEXT: movaps %xmm0, -56(%rsp)
-; SSE3-NEXT: movaps %xmm0, -72(%rsp)
; SSE3-NEXT: andl $3, %eax
; SSE3-NEXT: shll $3, %eax
+; SSE3-NEXT: movaps %xmm0, -56(%rsp)
+; SSE3-NEXT: movaps %xmm0, -72(%rsp)
; SSE3-NEXT: movsd -72(%rsp,%rax), %xmm0 # xmm0 = mem[0],zero
; SSE3-NEXT: movsd -40(%rsp,%rax), %xmm1 # xmm1 = mem[0],zero
; SSE3-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -2043,13 +2043,13 @@ define void @indices_convert() {
; SSSE3-LABEL: indices_convert:
; SSSE3: # %bb.0: # %bb
; SSSE3-NEXT: movaps (%rax), %xmm0
+; SSSE3-NEXT: movl (%rax), %eax
; SSSE3-NEXT: movaps %xmm0, -24(%rsp)
; SSSE3-NEXT: movaps %xmm0, -40(%rsp)
-; SSSE3-NEXT: movl (%rax), %eax
-; SSSE3-NEXT: movaps %xmm0, -56(%rsp)
-; SSSE3-NEXT: movaps %xmm0, -72(%rsp)
; SSSE3-NEXT: andl $3, %eax
; SSSE3-NEXT: shll $3, %eax
+; SSSE3-NEXT: movaps %xmm0, -56(%rsp)
+; SSSE3-NEXT: movaps %xmm0, -72(%rsp)
; SSSE3-NEXT: movsd -72(%rsp,%rax), %xmm0 # xmm0 = mem[0],zero
; SSSE3-NEXT: movsd -40(%rsp,%rax), %xmm1 # xmm1 = mem[0],zero
; SSSE3-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -2059,16 +2059,16 @@ define void @indices_convert() {
; SSE41-LABEL: indices_convert:
; SSE41: # %bb.0: # %bb
; SSE41-NEXT: movaps (%rax), %xmm0
-; SSE41-NEXT: extractps $2, %xmm0, %eax
+; SSE41-NEXT: extractps $3, %xmm0, %eax
+; SSE41-NEXT: extractps $2, %xmm0, %ecx
; SSE41-NEXT: movaps %xmm0, -24(%rsp)
; SSE41-NEXT: movaps %xmm0, -40(%rsp)
-; SSE41-NEXT: andl $3, %eax
-; SSE41-NEXT: extractps $3, %xmm0, %ecx
+; SSE41-NEXT: andl $3, %ecx
; SSE41-NEXT: movaps %xmm0, -56(%rsp)
; SSE41-NEXT: movaps %xmm0, -72(%rsp)
-; SSE41-NEXT: andl $3, %ecx
-; SSE41-NEXT: movsd -72(%rsp,%rcx,8), %xmm0 # xmm0 = mem[0],zero
-; SSE41-NEXT: movsd -40(%rsp,%rax,8), %xmm1 # xmm1 = mem[0],zero
+; SSE41-NEXT: andl $3, %eax
+; SSE41-NEXT: movsd -72(%rsp,%rax,8), %xmm0 # xmm0 = mem[0],zero
+; SSE41-NEXT: movsd -40(%rsp,%rcx,8), %xmm1 # xmm1 = mem[0],zero
; SSE41-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; SSE41-NEXT: movups %xmm1, (%rax)
; SSE41-NEXT: retq
>From 0f3a352f35c58794774db17b73a06cd6c570c9f7 Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Thu, 6 Aug 2026 18:28:53 +0800
Subject: [PATCH 4/4] fix adjustWritemask
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 158 +++++++++++++---------
1 file changed, 96 insertions(+), 62 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 24935ce123c3d..789e3969b3f66 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19163,19 +19163,27 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
}
+/// An EXTRACT_SUBREG reading \p NumLanes consecutive registers of a MIMG
+/// result, starting at \p FirstLane.
+struct MIMGLaneUse {
+ SDNode *User;
+ unsigned FirstLane;
+ unsigned NumLanes;
+};
+
/// Helper function for adjustWritemask. Collect the EXTRACT_SUBREG users of
-/// \p Node that read a single 32-bit register into \p Lanes, together with the
-/// lane they read. \p LaneOffset is the lane \p Node itself starts at.
+/// \p Node into \p Lanes, together with the range of lanes each one reads.
+/// \p LaneOffset is the lane \p Node itself starts at.
///
-/// EXTRACT_SUBREGs that name a wider piece are looked through, since they only
-/// rename a range of lanes; the reads of the individual registers are what
-/// decides which channels are live.
+/// An EXTRACT_SUBREG naming several registers is first looked through, since
+/// the reads of the individual registers describe the live channels more
+/// precisely. Only when it is used as a unit is it recorded as spanning a
+/// range.
///
/// \returns false if any use cannot be described as a lane read.
-static bool
-collectMIMGLaneUsers(SDNode *Node, unsigned LaneOffset,
- const SIRegisterInfo &TRI,
- SmallVectorImpl<std::pair<SDNode *, unsigned>> &Lanes) {
+static bool collectMIMGLaneUsers(SDNode *Node, unsigned LaneOffset,
+ const SIRegisterInfo &TRI,
+ SmallVectorImpl<MIMGLaneUse> &Lanes) {
for (SDUse &Use : Node->uses()) {
// Don't look at users of the chain.
if (Use.getResNo() != 0)
@@ -19192,13 +19200,19 @@ collectMIMGLaneUsers(SDNode *Node, unsigned LaneOffset,
if (Offset % 32 != 0 || Size % 32 != 0)
return false;
- if (Size == 32) {
- Lanes.emplace_back(User, LaneOffset + Offset / 32);
+ unsigned FirstLane = LaneOffset + Offset / 32;
+ unsigned NumLanes = Size / 32;
+ if (NumLanes == 1) {
+ Lanes.push_back({User, FirstLane, 1});
continue;
}
- if (!collectMIMGLaneUsers(User, LaneOffset + Offset / 32, TRI, Lanes))
- return false;
+ size_t Rollback = Lanes.size();
+ if (collectMIMGLaneUsers(User, FirstLane, TRI, Lanes))
+ continue;
+
+ Lanes.truncate(Rollback);
+ Lanes.push_back({User, FirstLane, NumLanes});
}
return true;
@@ -19239,23 +19253,36 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
}
// Try to figure out the used register components
- SmallVector<std::pair<SDNode *, unsigned>, 8> LaneUses;
+ SmallVector<MIMGLaneUse, 8> LaneUses;
if (!collectMIMGLaneUsers(Node, 0, *Subtarget->getRegisterInfo(), LaneUses))
return Node;
- for (auto [User, UserLane] : LaneUses) {
- // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
- // Note that subregs are packed, i.e. Lane==0 is the first bit set
- // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
- // set, etc.
- Lane = UserLane;
- if (Lane >= std::size(Users))
+ for (const MIMGLaneUse &LU : LaneUses) {
+ if (LU.FirstLane + LU.NumLanes > std::size(Users))
return Node;
- // Check if the use is for the TFE/LWE generated result at VGPRn+1.
- if (UsesTFC && Lane == TFCLane) {
- Users[Lane] = User;
- } else {
+ // The TFE/LWE result is not a texture channel, so a read covering several
+ // lanes at once cannot be described as reading channels if it also takes
+ // that one in.
+ if (UsesTFC && LU.NumLanes > 1 && LU.FirstLane <= TFCLane &&
+ TFCLane < LU.FirstLane + LU.NumLanes)
+ return Node;
+
+ for (Lane = LU.FirstLane; Lane != LU.FirstLane + LU.NumLanes; ++Lane) {
+ // Abort if we have more than one user per component.
+ if (Users[Lane])
+ return Node;
+ Users[Lane] = LU.User;
+
+ // Check if the use is for the TFE/LWE generated result at VGPRn+1.
+ if (UsesTFC && Lane == TFCLane)
+ continue;
+
+ // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
+ // Note that subregs are packed, i.e. Lane==0 is the first bit set
+ // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
+ // set, etc.
+ //
// Set which texture component corresponds to the lane.
unsigned Comp;
for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
@@ -19263,11 +19290,6 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
Dmask &= ~(1 << Comp);
}
- // Abort if we have more than one user per component.
- if (Users[Lane])
- return Node;
-
- Users[Lane] = User;
NewDmask |= 1 << Comp;
}
}
@@ -19289,6 +19311,26 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
if (NewDmask == OldDmask)
return Node;
+ // Work out where each lane ends up once the unused channels are dropped.
+ // Lane order is channel order, so the lanes that survive keep their relative
+ // positions, which means a run of adjacent lanes stays adjacent and can
+ // still be named by a single sub register index.
+ unsigned NewLaneOf[std::size(Users)];
+ for (unsigned I = 0, New = 0; I != std::size(Users); ++I) {
+ NewLaneOf[I] = New;
+ // NoChannels turned an arbitrary channel on below, and it takes up lane 0
+ // even though nothing reads it.
+ if (Users[I] || (I == 0 && NoChannels))
+ ++New;
+ }
+
+ // Give up if any of the reads cannot be named in its new position.
+ for (const MIMGLaneUse &LU : LaneUses) {
+ if (SIRegisterInfo::getSubRegFromChannelOrNone(
+ NewLaneOf[LU.FirstLane], LU.NumLanes) == AMDGPU::NoSubRegister)
+ return Node;
+ }
+
unsigned BitsSet = llvm::popcount(NewDmask);
// Check for TFE or LWE - increase the number of channels by one to account
@@ -19329,46 +19371,38 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
}
if (NewChannels == 1) {
- assert(Node->hasNUsesOfValue(1, 0));
+ // One channel and no TFE/LWE result, so exactly one lane is read.
+ assert(Node->hasNUsesOfValue(1, 0) && LaneUses.size() == 1 &&
+ LaneUses[0].NumLanes == 1);
+ SDNode *User = LaneUses[0].User;
SDNode *Copy =
DAG.getMachineNode(TargetOpcode::COPY, SDLoc(Node),
- Users[Lane]->getValueType(0), SDValue(NewNode, 0));
- DAG.ReplaceAllUsesWith(Users[Lane], Copy);
+ User->getValueType(0), SDValue(NewNode, 0));
+ DAG.ReplaceAllUsesWith(User, Copy);
return nullptr;
}
// Update the users of the node with the new indices
- for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
- SDNode *User = Users[i];
- if (!User) {
- // Handle the special case of NoChannels. We set NewDmask to 1 above, but
- // Users[0] is still nullptr because channel 0 doesn't really have a use.
- if (i || !NoChannels)
- continue;
- } else {
- SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
- SDNode *NewUser = DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op);
- if (NewUser != User) {
- DAG.ReplaceAllUsesWith(SDValue(User, 0), SDValue(NewUser, 0));
- DAG.RemoveDeadNode(User);
- }
+ for (const MIMGLaneUse &LU : LaneUses) {
+ unsigned NewFirstLane = NewLaneOf[LU.FirstLane];
+
+ // A read that now spans the whole result is no longer a sub register
+ // reference, since no index names a register in its entirety.
+ if (NewFirstLane == 0 && LU.NumLanes == NewChannels) {
+ SDNode *Copy =
+ DAG.getMachineNode(TargetOpcode::COPY, SDLoc(Node),
+ LU.User->getValueType(0), SDValue(NewNode, 0));
+ DAG.ReplaceAllUsesWith(LU.User, Copy);
+ continue;
}
- switch (Idx) {
- default:
- break;
- case AMDGPU::sub0:
- Idx = AMDGPU::sub1;
- break;
- case AMDGPU::sub1:
- Idx = AMDGPU::sub2;
- break;
- case AMDGPU::sub2:
- Idx = AMDGPU::sub3;
- break;
- case AMDGPU::sub3:
- Idx = AMDGPU::sub4;
- break;
+ unsigned Idx =
+ SIRegisterInfo::getSubRegFromChannel(NewFirstLane, LU.NumLanes);
+ SDValue Op = DAG.getTargetConstant(Idx, SDLoc(LU.User), MVT::i32);
+ SDNode *NewUser = DAG.UpdateNodeOperands(LU.User, SDValue(NewNode, 0), Op);
+ if (NewUser != LU.User) {
+ DAG.ReplaceAllUsesWith(SDValue(LU.User, 0), SDValue(NewUser, 0));
+ DAG.RemoveDeadNode(LU.User);
}
}
More information about the llvm-commits
mailing list