[llvm] [PeepholeOptimizer] Add REG_SEQUENCE subregister use folding (PR #205795)
Frederik Harwath via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 07:50:22 PDT 2026
https://github.com/frederik-h updated https://github.com/llvm/llvm-project/pull/205795
>From f85bcc5ab1c53ce94929319cf2ea8998a0a8cdc4 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 15 May 2026 10:29:36 -0400
Subject: [PATCH 1/6] [PeepholeOptimizer] Fold REG_SEQUENCE subregister uses
Rewrite uses of REG_SEQUENCE subregisters to directly use the source
registers when safe. This exposes patterns for subsequent optimizations
like AMDGPU si-peephole-sdwa that would otherwise be obscured and can
reduce register pressure.
Example transformation:
%2 = REG_SEQUENCE %0, sub0, %1, sub1
%3 = V_ADD %2.sub0, ...
becomes:
%3 = V_ADD %0, ...
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 15 +
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 150 ++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 6 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 288 +++++-----
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 38 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 142 +++--
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 20 +-
llvm/test/CodeGen/AMDGPU/and_or.ll | 58 +-
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 124 ++---
llvm/test/CodeGen/AMDGPU/ds_write2.ll | 46 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 120 ++---
.../CodeGen/AMDGPU/image-sample-waterfall.ll | 8 +-
llvm/test/CodeGen/AMDGPU/indirect-call.ll | 24 +-
.../AMDGPU/insert_vector_elt.v2bf16.ll | 12 +-
.../CodeGen/AMDGPU/insert_vector_elt.v2i16.ll | 9 +-
.../AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll | 17 +-
.../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll | 15 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 21 +-
.../AMDGPU/llvm.amdgcn.tensor.load.store.ll | 268 +++-------
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 504 +++++++++---------
llvm/test/CodeGen/AMDGPU/load-global-i16.ll | 112 ++--
llvm/test/CodeGen/AMDGPU/load-global-i8.ll | 22 +-
llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 204 +++----
...uf-legalize-operands-non-ptr-intrinsics.ll | 176 +++---
llvm/test/CodeGen/AMDGPU/optimize-compare.mir | 4 +-
llvm/test/CodeGen/AMDGPU/or3.ll | 16 +-
.../peephole-opt-fold-reg-sequence-subreg.mir | 13 +-
.../AMDGPU/peephole-opt-regseq-removal.mir | 5 -
.../CodeGen/AMDGPU/peephole-regseq-fold.mir | 309 +++++++++++
.../AMDGPU/sdwa-peephole-reg-sequence.mir | 114 ++++
llvm/test/CodeGen/AMDGPU/sibling-call.ll | 18 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 72 +--
llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll | 8 +-
.../CodeGen/AMDGPU/waterfall_kills_scc.ll | 23 +-
llvm/test/CodeGen/Thumb2/mve-clmul.ll | 411 +++++++-------
.../Thumb2/mve-laneinterleaving-cost.ll | 46 +-
.../CodeGen/Thumb2/mve-laneinterleaving.ll | 44 +-
llvm/test/CodeGen/Thumb2/mve-vabdus.ll | 18 +-
llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll | 28 +-
.../CodeGen/X86/peephole-reg-sequence.mir | 28 +
45 files changed, 2058 insertions(+), 1543 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/peephole-regseq-fold.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/sdwa-peephole-reg-sequence.mir
create mode 100644 llvm/test/CodeGen/X86/peephole-reg-sequence.mir
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 03f3bf26d0608..d0bd66fce7508 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -614,6 +614,21 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
: RegSubRegPair(Reg, SubReg), SubIdx(SubIdx) {}
};
+ /// Check if \p MO can be used as operand \p OpIdx for \p MI.
+ ///
+ /// \param MI The instruction to validate against.
+ /// \param OpIdx The operand position to check.
+ /// \param MO The operand to validate. Does not need to have a parent.
+ /// If nullptr, checks MI.getOperand(OpIdx).
+ ///
+ /// \returns true if the operand would be legal at the given position.
+ ///
+ /// \note The generic implementation returns true.
+ virtual bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
+ const MachineOperand *MO) const {
+ return true;
+ }
+
/// Build the equivalent inputs of a REG_SEQUENCE for the given \p MI
/// and \p DefIdx.
/// \p [out] InputRegs of the equivalent REG_SEQUENCE. Each element of
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index ec8a0336a2105..f303fb2e3b6c1 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -63,6 +63,21 @@
// =>
// b = bitcast A <-- cross-bank copy
// C = copy A <-- same-bank copy
+//
+// - Fold REG_SEQUENCE subregister uses:
+//
+// Replace uses of subregisters of registers defined by
+// REG_SEQUENCE instructions with direct uses of the source
+// registers.
+//
+// Example:
+// A = REG_SEQUENCE B, sub0, C, sub1
+// D = ADD A.sub0, E
+// =>
+// A = REG_SEQUENCE B, sub0, C, sub1
+// D = ADD B, E
+//
+// Eliminate the REG_SEQUENCE if all uses are folded.
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/PeepholeOptimizer.h"
@@ -121,6 +136,10 @@ static cl::opt<bool> DisableNAPhysCopyOpt(
"disable-non-allocatable-phys-copy-opt", cl::Hidden, cl::init(false),
cl::desc("Disable non-allocatable physical register copy optimization"));
+static cl::opt<bool> DisablePeepholeRegSeqFold(
+ "disable-peephole-regseq-fold", cl::Hidden, cl::init(false),
+ cl::desc("Disable REG_SEQUENCE subregister use folding"));
+
// Limit the number of PHI instructions to process
// in PeepholeOptimizer::getNextSource.
static cl::opt<unsigned>
@@ -142,6 +161,10 @@ STATISTIC(NumSelects, "Number of selects optimized");
STATISTIC(NumUncoalescableCopies, "Number of uncoalescable copies optimized");
STATISTIC(NumRewrittenCopies, "Number of copies rewritten");
STATISTIC(NumNAPhysCopies, "Number of non-allocatable physical copies removed");
+STATISTIC(NumRegSeqUsesRewritten,
+ "Number of REG_SEQUENCE subregister uses rewritten");
+STATISTIC(NumRegSeqEliminated,
+ "Number of REG_SEQUENCE instructions eliminated");
namespace {
@@ -457,6 +480,8 @@ class PeepholeOptimizer : private MachineFunction::Delegate {
bool optimizeUncoalescableCopy(MachineInstr &MI,
SmallPtrSetImpl<MachineInstr *> &LocalMIs);
bool optimizeRecurrence(MachineInstr &PHI);
+ bool foldRegSequenceUses(MachineInstr &MI, bool &ShouldEraseMI);
+ bool canRewriteRegSequenceSubRegUse(MachineOperand &UseMO, RegSubRegPair Src);
bool findNextSource(const TargetRegisterClass *DefRC, unsigned DefSubReg,
RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap);
bool isMoveImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
@@ -1739,6 +1764,119 @@ bool PeepholeOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
return Impl.run(MF);
}
+/// Check if a REG_SEQUENCE subregister use can be safely rewritten to use
+/// the source register directly.
+bool PeepholeOptimizer::canRewriteRegSequenceSubRegUse(MachineOperand &UseMO,
+ RegSubRegPair Src) {
+ if (!Src.Reg.isValid() || !Src.Reg.isVirtual())
+ return false;
+
+ MachineInstr *UseMI = UseMO.getParent();
+ Register DstReg = UseMO.getReg();
+
+ // Instructions, e.g. AMDGPU S_MOVRELS_B32, may require specific
+ // subregister relationships with implicit operands. Replacing the
+ // REG_SEQUENCE use with one of the source of its subregisters can
+ // violate these constraints.
+ if (UseMI->hasRegisterImplicitUseOperand(DstReg) || UseMI->isInlineAsm())
+ return false;
+
+ const MCInstrDesc &UseDesc = UseMI->getDesc();
+ unsigned OpNo = UseMI->getOperandNo(&UseMO);
+ if (OpNo >= UseDesc.getNumOperands()) {
+ assert(UseDesc.isVariadic() &&
+ "Explicit op beyond getNumOperands() implies UseMI variadic");
+ // No target specific legality check possible/necessary.
+ return true;
+ }
+
+ const TargetRegisterClass *SrcRC = MRI->getRegClass(Src.Reg);
+ const TargetRegisterClass *OpRC = TII->getRegClass(UseDesc, OpNo);
+
+ if (OpRC) {
+ const TargetRegisterClass *ConstrainRC =
+ Src.SubReg ? TRI->getMatchingSuperRegClass(SrcRC, OpRC, Src.SubReg)
+ : OpRC;
+ if (!ConstrainRC || !MRI->constrainRegClass(Src.Reg, ConstrainRC))
+ return false;
+ }
+
+ // Check target-specific operand legality on a temporary MO.
+ // Kill/undef flags do not reflect the actual MO, but this should
+ // not matter for legality.
+ MachineOperand MO = MachineOperand::CreateReg(Src.Reg, /*isDef=*/false);
+ MO.setSubReg(Src.SubReg);
+ if (TII->isOperandLegal(*UseMI, OpNo, &MO))
+ return true;
+
+ MRI->setRegClass(Src.Reg, SrcRC); // Undo constrainRegClass.
+ return false;
+}
+
+/// Fold uses of REG_SEQUENCE subregisters to directly use the source registers.
+///
+/// Example: %8 = V_ADD_CO_U32_e32 %4, %7.sub0
+/// where %7 = REG_SEQUENCE %5, sub0, %6, sub1
+/// becomes: %8 = V_ADD_CO_U32_e32 %4, %5
+bool PeepholeOptimizer::foldRegSequenceUses(MachineInstr &MI,
+ bool &ShouldEraseMI) {
+ assert(MI.isRegSequence() && "Expected REG_SEQUENCE instruction");
+ ShouldEraseMI = false;
+
+ if (DisablePeepholeRegSeqFold)
+ return false;
+
+ Register DstReg = MI.getOperand(0).getReg();
+
+ bool Changed = false;
+ for (auto UI = MRI->use_nodbg_begin(DstReg), UE = MRI->use_nodbg_end();
+ UI != UE;) {
+ MachineOperand &UseMO = *UI;
+ ++UI; // advance before modifying ops to avoid invalidation
+
+ unsigned UseSubReg = UseMO.getSubReg();
+ if (UseSubReg == 0 || UseMO.isUndef())
+ continue;
+
+ // Find the matching source in the REG_SEQUENCE operands.
+ // REG_SEQUENCE format: dst, src0, subidx0, src1, subidx1, ...
+ MachineOperand *SrcOp = nullptr;
+ for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
+ if (MI.getOperand(i + 1).getImm() == UseSubReg) {
+ SrcOp = &MI.getOperand(i);
+ break;
+ }
+ }
+
+ if (!SrcOp)
+ continue;
+
+ Register SrcReg = SrcOp->getReg();
+ unsigned SrcSubReg = SrcOp->getSubReg();
+ if (!canRewriteRegSequenceSubRegUse(UseMO, {SrcReg, SrcSubReg}))
+ continue;
+
+ SrcOp->setIsKill(false);
+ MRI->clearKillFlags(SrcReg);
+
+ UseMO.setReg(SrcReg);
+ UseMO.setSubReg(SrcSubReg);
+ UseMO.setIsUndef(SrcOp->isUndef());
+ UseMO.setIsKill(false);
+
+ Changed = true;
+ ++NumRegSeqUsesRewritten;
+ }
+
+ if (Changed && MRI->use_nodbg_empty(DstReg)) {
+ MRI->markUsesInDebugValueAsUndef(DstReg);
+ ShouldEraseMI = true;
+ ++NumRegSeqEliminated;
+ }
+
+ return Changed;
+}
+
bool PeepholeOptimizer::run(MachineFunction &MF) {
LLVM_DEBUG(dbgs() << "********** PEEPHOLE OPTIMIZER **********\n");
@@ -1868,6 +2006,18 @@ bool PeepholeOptimizer::run(MachineFunction &MF) {
continue;
}
+ if (MI->isRegSequence()) {
+ bool ShouldEraseMI;
+ if (foldRegSequenceUses(*MI, ShouldEraseMI)) {
+ if (ShouldEraseMI) {
+ LocalMIs.erase(MI);
+ MI->eraseFromParent();
+ }
+ Changed = true;
+ continue;
+ }
+ }
+
if (MI->isCopy() && (foldRedundantCopy(*MI) ||
foldRedundantNAPhysCopy(*MI, NAPhysToVirtMIs))) {
LocalMIs.erase(MI);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 48953cee43956..d1562bf331115 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6281,7 +6281,8 @@ void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
const MCOperandInfo &OpInfo,
- const MachineOperand &MO) const {
+ const MachineOperand &MO,
+ const MachineFunction *MF) const {
if (!MO.isReg())
return false;
@@ -6294,7 +6295,13 @@ bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
const TargetRegisterClass *RC = MRI.getRegClass(Reg);
if (MO.getSubReg()) {
- const MachineFunction *MF = MO.getParent()->getMF();
+ if (!MF) {
+ const MachineInstr *Parent = MO.getParent();
+ if (!Parent)
+ return false;
+ MF = Parent->getMF();
+ }
+
const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF);
if (!SuperRC)
return false;
@@ -6325,7 +6332,7 @@ bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
}
}
- if (!isLegalRegOperand(MRI, OpInfo, MO))
+ if (!isLegalRegOperand(MRI, OpInfo, MO, MI.getMF()))
return false;
// check Accumulate GPR operand
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 1d67c8664ff44..18c38626e4826 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1463,7 +1463,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// Check if \p MO is a legal operand if it was the \p OpIdx Operand
/// for \p MI.
bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
- const MachineOperand *MO = nullptr) const;
+ const MachineOperand *MO = nullptr) const override;
/// Check if \p MO would be a valid operand for the given operand
/// definition \p OpInfo. Note this does not attempt to validate constant bus
@@ -1476,8 +1476,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// given operand description or operand index.
/// The operand index version provide more legality checks
bool isLegalRegOperand(const MachineRegisterInfo &MRI,
- const MCOperandInfo &OpInfo,
- const MachineOperand &MO) const;
+ const MCOperandInfo &OpInfo, const MachineOperand &MO,
+ const MachineFunction *MF = nullptr) const;
bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
const MachineOperand &MO) const;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2d5fc5cf22425..14660b524a6d5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -154920,23 +154920,23 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v152, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x2
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_b32 v71, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v81, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr141_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr139_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr138_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr137_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr136_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
@@ -154944,40 +154944,40 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr136_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr142_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr137_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr138_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr152_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr143_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr140_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
@@ -154997,7 +154997,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
@@ -155010,10 +155010,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
@@ -155021,12 +155021,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
@@ -155049,48 +155049,48 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[133:134], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[96:97], 24, v[29:30]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[101:102], 24, v[23:24]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v15
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v109, 24, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v111, 8, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v120, 8, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v121, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v122, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v136, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v138, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v127, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v137, 24, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v139, 8, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v141, 8, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 24, v71
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v71
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v81
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v81
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 8, v70
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 24, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 8, v80
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 24, v30
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 8, v30
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v29
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 24, v28
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 24, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v26
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v25
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v89, 24, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v91, 8, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v92, 8, v23
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 24, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v22
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v22
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v108, 8, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v110, 24, v20
@@ -155099,10 +155099,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v125, 24, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v126, 8, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v140, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[86:87], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[84:85], 24, v[15:16]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[112:113], 24, v[11:12]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[134:135], 24, v[1:2]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[82:83], 24, v[70:71]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[82:83], 24, v[80:81]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[97:98], 24, v[25:26]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[102:103], 24, v[21:22]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[118:119], 24, v[19:20]
@@ -155115,25 +155115,25 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v3.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v5.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v6.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.h, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v7.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.h, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v9.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.h, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.h, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.h, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.h, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.h, v8.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.h, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v136.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v105.h, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v106.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v12.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v142.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v13.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v137.h, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v138.h, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v14.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v152.h, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v15.h
@@ -155167,10 +155167,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.h, v29.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.h, v30.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v30.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v176.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.h, v70.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.h, v71.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v71.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v176.h, v80.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.h, v80.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v81.h
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
@@ -155186,7 +155186,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81
; GFX11-TRUE16-NEXT: .LBB90_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB90_4
@@ -155308,7 +155308,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 24, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v36
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v36
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v151, v19, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -155372,8 +155372,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v30
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v49
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 24, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v108, 8, v35
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v163, v19, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -155404,14 +155404,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v71
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v81
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 24, v51
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v51
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v165, v19, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v71
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v81
; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v53, v17, v24
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
@@ -155420,14 +155420,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v70
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v37.l, v160.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v80
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v53.l, v165.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v166, v17, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v70
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v80
; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_cndmask_b32 v52, v19, v24
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
@@ -155436,9 +155436,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 24, v53
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v37.l, v160.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 24, v53
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 8, v53
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v92, 8, v37
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v167, v19, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
@@ -155453,8 +155453,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v55.l, v167.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v92, 8, v37
; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v176, v17, v22, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
@@ -155468,7 +155469,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v20, 0x40c00000, v20
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 24, v55
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v55
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v55
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v20, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
@@ -155504,7 +155505,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.l, v162.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v138, 24, v65
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v137, 24, v65
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v179, v2, v19, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v17, 16, 1
@@ -155536,7 +155537,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v139, 8, v65
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v50
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v50
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v48
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v181, v2, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
@@ -155553,7 +155554,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v69.l, v181.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v182, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v41, v2, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v6, 0x7fff
@@ -155566,14 +155567,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v68.l, v182.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v41.h
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v46, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v81, v1, v8
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v71, v1, v8
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
@@ -155583,8 +155584,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v52.l, v166.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v44.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v80, v1, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v46.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v70, v1, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v9
@@ -155597,15 +155598,15 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v12
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v62.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v63, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v62.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v77, v1, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v2, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
@@ -155613,31 +155614,30 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v63.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v77.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v54.l, v176.h
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[130:131], 24, v[68:69]
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v84, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v86, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[96:97], 24, v[52:53]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[97:98], 24, v[48:49]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[82:83], 24, v[54:55]
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v94, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v104, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v94.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v105, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v104.h
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v106, v1, v2 :: v_dual_lshlrev_b32 v1, 16, v11
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[116:117], 24, v[84:85]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[116:117], 24, v[86:87]
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v3, v7, vcc_lo
@@ -155645,8 +155645,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v105.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[117:118], 24, v[80:81]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v106.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[117:118], 24, v[70:71]
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
@@ -155657,7 +155657,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v64.l, v178.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v127, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v136, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
@@ -155671,12 +155671,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v137, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v138, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v137.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v138.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v128, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
@@ -155686,7 +155686,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v66.l, v180.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v127.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v114.l, v136.h
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v142, v4, v6, vcc_lo
@@ -155715,31 +155715,31 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v143.h
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[134:135], 24, v[64:65]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[82:83], 24, v[54:55]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[97:98], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[102:103], 24, v[35:36]
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v144, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v152.h
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[118:119], 24, v[33:34]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[131:132], 24, v[31:32]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v145
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 8, v145
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[86:87], 24, v[144:145]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v144
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 24, v145
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v145
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[84:85], 24, v[144:145]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v144
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v129
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v129
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 8, v128
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v115
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v115
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v114
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v85
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v85
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v84
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v81
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v81
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v80
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 24, v115
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v115
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v114
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v87
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v87
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v86
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v71
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v71
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v70
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v111, 8, v69
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v120, 8, v68
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v136, 8, v66
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v127, 8, v66
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v141, 8, v64
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 8, v54
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v52
@@ -155753,13 +155753,13 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v141, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v139, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v138, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v137, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v67.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v7.l, v182.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v41.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v68.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v4, 16, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v5, v136, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v5, v127, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v5.l, v179.h
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v6, v121, 0xc0c0004
@@ -155769,36 +155769,36 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v5, v122, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v9.l, v181.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v69.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v81.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v71.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v9, v111, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v10, v109, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v62.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v80.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v44.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v84.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v70.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v46.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v86.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v8, v104, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v8, v95, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v9, v117, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v95, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v94, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v11, v93, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v94.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v63.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v104.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v77.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v114.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v11, v90, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v12, v116, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v13, v88, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v85.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v127.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v87.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v13.l, v136.h
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v105.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v106.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v115.h
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v78, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v75, 0xc0c0004
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
@@ -155807,7 +155807,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v14, 16, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v6.l, v128.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v7.l, v137.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v7.l, v138.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v8.l, v129.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v9.l, v152.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v10.l, v144.h
@@ -155815,16 +155815,16 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v12.l, v145.h
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v13.l, v146.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v31.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v74, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v72, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v63, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v5, v59, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v6, v99, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v7, v58, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v8, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v9, v46, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v10, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v11, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v9, v45, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v10, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v11, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v182, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v140, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v16, 16, v15
@@ -155852,7 +155852,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v108, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v102, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v107, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v106, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v105, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v12, v15, 16, v14
@@ -155874,9 +155874,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v89, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v79, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v77, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v22, v75, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v23, v73, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v22, v74, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v23, v72, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v24, v24, v100, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v15, v16, 16, v15
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v16, v18, 16, v17
@@ -155898,11 +155898,11 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v22, v22, v57, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v23, v23, v96, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v24, v24, v47, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v25, v45, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v25, v44, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v26, v26, v43, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v27, v27, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v28, v28, v42, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v29, v40, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v29, v183, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v20, v21, 16, v20
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v23, 16, v22
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v22, v25, 16, v24
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 7e231f48de12a..a751af1b67617 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -24163,13 +24163,13 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
@@ -24179,22 +24179,22 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[2:3]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[0:1]
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v0.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v1.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v2.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v3.h
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
@@ -24263,35 +24263,35 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[16:17]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v7, v13, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v3, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v12.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v12.h
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[20:21]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[21:22]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v21
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v17.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v20.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v21.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v21.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
index c09389ef700ac..1791bf59af12b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
@@ -429,7 +429,7 @@ define <10 x i16> @bitcast_v5i32_to_v10i16(<5 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; SI-NEXT: .LBB4_4: ; %end
@@ -1019,7 +1019,7 @@ define <10 x half> @bitcast_v5i32_to_v10f16(<5 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; SI-NEXT: .LBB8_4: ; %end
@@ -1648,7 +1648,7 @@ define <10 x i16> @bitcast_v5f32_to_v10i16(<5 x float> %a, i32 %b) #0 {
; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; SI-NEXT: .LBB12_4: ; %end
@@ -2253,7 +2253,7 @@ define <10 x half> @bitcast_v5f32_to_v10f16(<5 x float> %a, i32 %b) #0 {
; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; SI-NEXT: .LBB16_4: ; %end
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
index c66f5ea8e6c60..609d14e386544 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
@@ -484,7 +484,7 @@ define <14 x i16> @bitcast_v7i32_to_v14i16(<7 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
@@ -1196,7 +1196,7 @@ define <14 x half> @bitcast_v7i32_to_v14f16(<7 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
@@ -1961,7 +1961,7 @@ define <14 x i16> @bitcast_v7f32_to_v14i16(<7 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
@@ -2685,7 +2685,7 @@ define <14 x half> @bitcast_v7f32_to_v14f16(<7 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
index ead5d76b2e572..59969f18bb145 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
@@ -537,7 +537,7 @@ define <18 x i16> @bitcast_v9i32_to_v18i16(<9 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v14, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
@@ -1368,7 +1368,7 @@ define <18 x half> @bitcast_v9i32_to_v18f16(<9 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v14, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
@@ -2267,7 +2267,7 @@ define <18 x i16> @bitcast_v9f32_to_v18i16(<9 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v14, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
@@ -3127,7 +3127,7 @@ define <18 x half> @bitcast_v9f32_to_v18f16(<9 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v14, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
index 79c9fc7faf339..2500fbac40aa0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
@@ -596,7 +596,7 @@ define <22 x i16> @bitcast_v11i32_to_v22i16(<11 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v15, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v18, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v5
@@ -1549,7 +1549,7 @@ define <22 x half> @bitcast_v11i32_to_v22f16(<11 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v15, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v18, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v5
@@ -2583,7 +2583,7 @@ define <22 x i16> @bitcast_v11f32_to_v22i16(<11 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v15, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v18, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v5
@@ -3561,7 +3561,7 @@ define <22 x half> @bitcast_v11f32_to_v22f16(<11 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v15, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v18, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v5
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 092528a4feb1e..5e3daa5e2426b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -86388,18 +86388,18 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16
@@ -86408,9 +86408,9 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16
@@ -86428,15 +86428,15 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[5:6]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 24, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v8
@@ -86469,13 +86469,13 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v9.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.h, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v12.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v13.h
@@ -86651,7 +86651,7 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v66.h
@@ -86659,114 +86659,112 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v67.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v24
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v68.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v81.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v71, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v8 :: v_dual_lshlrev_b32 v5, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v1, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v82.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v26
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v71.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v81.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v27, v2, v3 :: v_dual_add_f32 v2, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v98, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v28
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v97.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v99, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v99.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v98.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v9 :: v_dual_lshlrev_b32 v3, 16, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v99.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[25:26]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[23:24]
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v5 :: v_dual_add_f32 v6, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[23:24]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[21:22]
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v115, v4, v6 :: v_dual_add_f32 v6, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v115.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v117, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[21:22]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v33
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v117, v2, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v32
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v27
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v115.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v33
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v118, v7, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v27
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v4, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v117.h
@@ -86777,7 +86775,7 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[30:31], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v48
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v48
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v53.h
@@ -86814,11 +86812,11 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v23.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v81.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v82.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v25.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v68.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v97.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v98.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v27.h
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v7, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v8, v36, 0xc0c0004
@@ -86826,7 +86824,7 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v10, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v11, v100, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v98, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v96, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v87, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v34, 0xc0c0004
@@ -86834,7 +86832,7 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v10, v14, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v71.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v81.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v28.h
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v11, v16, 16, v15
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v115.h
@@ -86850,8 +86848,8 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v84, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v15, v30, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v71, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v20, v70, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v21, v21, v69, 0xc0c0004
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index a1c0db086bf3c..1be677ad46306 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -398,9 +398,9 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
-; SI-NEXT: v_alignbit_b32 v11, v0, v8, 24
-; SI-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 8
+; SI-NEXT: v_alignbit_b32 v11, s4, v8, 24
+; SI-NEXT: v_alignbit_b32 v10, s4, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 8
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
@@ -2556,7 +2556,7 @@ define <6 x half> @bitcast_v3i32_to_v6f16(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -3051,7 +3051,7 @@ define <6 x i16> @bitcast_v3i32_to_v6i16(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -3543,9 +3543,9 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
-; SI-NEXT: v_alignbit_b32 v11, v0, v8, 24
-; SI-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 8
+; SI-NEXT: v_alignbit_b32 v11, s4, v8, 24
+; SI-NEXT: v_alignbit_b32 v10, s4, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 8
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
@@ -5730,7 +5730,7 @@ define <6 x half> @bitcast_v3f32_to_v6f16(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -6233,7 +6233,7 @@ define <6 x i16> @bitcast_v3f32_to_v6i16(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/and_or.ll b/llvm/test/CodeGen/AMDGPU/and_or.ll
index 460cc4bbadb02..2215fb80af9e3 100644
--- a/llvm/test/CodeGen/AMDGPU/and_or.ll
+++ b/llvm/test/CodeGen/AMDGPU/and_or.ll
@@ -2,7 +2,7 @@
;RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=fiji | FileCheck -check-prefix=VI %s
;RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 | FileCheck -check-prefix=GFX9 %s
;RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
-;RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck -check-prefix=GFX10 %s
+;RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck -check-prefixes=GFX10,GFX11-NODELAY %s
; ===================================================================================
; V_AND_OR_B32
@@ -191,12 +191,12 @@ define <2 x i32> @v_and_or_v2i32_b(<2 x i32> inreg %a, <2 x i32> %b, <2 x i32> i
; GFX9-NEXT: v_or_b32_e32 v0, s18, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_and_or_v2i32_b:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DAG: v_and_or_b32 v0, s{{[0-9]+}}, v0, s{{[0-9]+}}
-; GFX10-DAG: v_and_or_b32 v1, s{{[0-9]+}}, v1, s{{[0-9]+}}
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX11-NODELAY-LABEL: v_and_or_v2i32_b:
+; GFX11-NODELAY: ; %bb.0:
+; GFX11-NODELAY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NODELAY-NEXT: v_and_or_b32 v0, s0, v0, s2
+; GFX11-NODELAY-NEXT: v_and_or_b32 v1, s1, v1, s3
+; GFX11-NODELAY-NEXT: s_setpc_b64 s[30:31]
%x = and <2 x i32> %a, %b
%result = or <2 x i32> %x, %c
ret <2 x i32> %result
@@ -219,12 +219,12 @@ define <2 x i32> @v_and_or_v2i32_ab(<2 x i32> %a, <2 x i32> %b, <2 x i32> inreg
; GFX9-NEXT: v_and_or_b32 v0, v0, v2, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_and_or_v2i32_ab:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DAG: v_and_or_b32 v1, v1, v3, {{s[0-9]+}}
-; GFX10-DAG: v_and_or_b32 v0, v0, v2, {{s[0-9]+}}
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX11-NODELAY-LABEL: v_and_or_v2i32_ab:
+; GFX11-NODELAY: ; %bb.0:
+; GFX11-NODELAY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NODELAY-NEXT: v_and_or_b32 v0, v0, v2, s0
+; GFX11-NODELAY-NEXT: v_and_or_b32 v1, v1, v3, s1
+; GFX11-NODELAY-NEXT: s_setpc_b64 s[30:31]
%x = and <2 x i32> %a, %b
%result = or <2 x i32> %x, %c
ret <2 x i32> %result
@@ -272,9 +272,9 @@ define <2 x i32> @v_and_or_v2i32_inline_const(<2 x i32> %a, <2 x i32> %b) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x809
+; GFX9-NEXT: s_movk_i32 s5, 0x808
; GFX9-NEXT: v_and_or_b32 v1, v1, s4, v3
-; GFX9-NEXT: s_movk_i32 s4, 0x808
-; GFX9-NEXT: v_and_or_b32 v0, v0, s4, v2
+; GFX9-NEXT: v_and_or_b32 v0, v0, s5, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_and_or_v2i32_inline_const:
@@ -337,12 +337,13 @@ define <2 x i32> @v_and_or_v2i32_inline_const_x3(<2 x i32> %a) {
; GFX9-NEXT: v_or_b32_e32 v0, 0x81, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_and_or_v2i32_inline_const_x3:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 [[SR:s[0-9]+]], 0x808
-; GFX10-DAG: v_and_or_b32 v0, v0, [[SR]], 0x81
-; GFX10-DAG: v_and_or_b32 v1, 0x809, v1, 16
+; GFX11-NODELAY-LABEL: v_and_or_v2i32_inline_const_x3:
+; GFX11-NODELAY: ; %bb.0:
+; GFX11-NODELAY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NODELAY-NEXT: s_movk_i32 s0, 0x81
+; GFX11-NODELAY-NEXT: v_and_or_b32 v1, 0x809, v1, 16
+; GFX11-NODELAY-NEXT: v_and_or_b32 v0, 0x808, v0, s0
+; GFX11-NODELAY-NEXT: s_setpc_b64 s[30:31]
%x = and <2 x i32> %a, <i32 2056, i32 2057>
%result = or <2 x i32> %x, <i32 129, i32 16>
ret <2 x i32> %result
@@ -367,14 +368,15 @@ define <2 x i32> @v_and_or_v2i32_inline_const_x4(<2 x i32> %a) {
; GFX9-NEXT: v_or_b32_e32 v0, 0x81, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_and_or_v2i32_inline_const_x4:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DAG: s_movk_i32 [[SR0:s[0-9]+]], 0x808
-; GFX10-DAG: s_movk_i32 [[SR1:s[0-9]+]], 0x809
+; GFX11-NODELAY-LABEL: v_and_or_v2i32_inline_const_x4:
+; GFX11-NODELAY: ; %bb.0:
+; GFX11-NODELAY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NODELAY-NEXT: s_movk_i32 s0, 0x81
+; GFX11-NODELAY-NEXT: s_movk_i32 s1, 0x101
+; GFX11-NODELAY-NEXT: v_and_or_b32 v0, 0x808, v0, s0
+; GFX11-NODELAY-NEXT: v_and_or_b32 v1, 0x809, v1, s1
+; GFX11-NODELAY-NEXT: s_setpc_b64 s[30:31]
; GFX10-CHECK-NOT: {{.}}
-; GFX10-DAG: v_and_or_b32 v0, v0, [[SR0]], 0x81
-; GFX10-DAG: v_and_or_b32 v1, v1, [[SR1]], 0x101
%x = and <2 x i32> %a, <i32 2056, i32 2057>
%result = or <2 x i32> %x, <i32 129, i32 257>
ret <2 x i32> %result
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 93c0051a717c7..bc80b0acb291b 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -104,33 +104,33 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22
; SDAG-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v11, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22
; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc
-; SDAG-NEXT: v_mov_b32_e32 v10, 0
-; SDAG-NEXT: v_mov_b32_e32 v11, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, 0
+; SDAG-NEXT: v_mov_b32_e32 v9, 0
; SDAG-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB0_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[18:19], v30
+; SDAG-NEXT: v_lshr_b64 v[8:9], v[18:19], v30
; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v30
; SDAG-NEXT: v_lshl_b64 v[20:21], v[16:17], v20
-; SDAG-NEXT: v_or_b32_e32 v21, v11, v21
-; SDAG-NEXT: v_or_b32_e32 v20, v10, v20
+; SDAG-NEXT: v_or_b32_e32 v21, v9, v21
+; SDAG-NEXT: v_or_b32_e32 v20, v8, v20
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
-; SDAG-NEXT: v_subrev_i32_e64 v10, s[4:5], 64, v30
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v10
-; SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v21, vcc
+; SDAG-NEXT: v_subrev_i32_e64 v8, s[4:5], 64, v30
+; SDAG-NEXT: v_lshr_b64 v[8:9], v[16:17], v8
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v21, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v30
-; SDAG-NEXT: v_cndmask_b32_e64 v19, v11, v19, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v10, v10, v20, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v18, v10, v18, s[4:5]
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v30
-; SDAG-NEXT: v_cndmask_b32_e32 v23, 0, v11, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v22, 0, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v19, v9, v19, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v20, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v18, v8, v18, s[4:5]
+; SDAG-NEXT: v_lshr_b64 v[8:9], v[16:17], v30
+; SDAG-NEXT: v_cndmask_b32_e32 v23, 0, v9, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v22, 0, v8, vcc
; SDAG-NEXT: v_add_i32_e32 v34, vcc, -1, v28
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v29, vcc
; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v2, vcc
@@ -138,13 +138,13 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v11, 0
+; SDAG-NEXT: v_mov_b32_e32 v9, 0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: .LBB0_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v8, 31, v11
+; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v38, 31, v19
; SDAG-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
@@ -152,35 +152,35 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_add_i32_e32 v30, vcc, -1, v30
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_or_b32_e32 v9, v21, v9
-; SDAG-NEXT: v_or_b32_e32 v8, v20, v8
+; SDAG-NEXT: v_or_b32_e32 v11, v21, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v20, v10
; SDAG-NEXT: v_or_b32_e32 v22, v22, v38
; SDAG-NEXT: v_or_b32_e32 v18, v18, v39
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v10
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v8
; SDAG-NEXT: v_or_b32_e32 v1, v17, v1
-; SDAG-NEXT: v_sub_i32_e32 v10, vcc, v34, v18
-; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v35, v19, vcc
+; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v34, v18
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v35, v19, vcc
; SDAG-NEXT: v_or_b32_e32 v21, v31, v33
; SDAG-NEXT: v_or_b32_e32 v20, v30, v32
-; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v36, v22, vcc
-; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v37, v23, vcc
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v36, v22, vcc
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v37, v23, vcc
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[20:21]
-; SDAG-NEXT: v_ashrrev_i32_e32 v10, 31, v10
+; SDAG-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_and_b32_e32 v20, v10, v29
-; SDAG-NEXT: v_and_b32_e32 v21, v10, v28
-; SDAG-NEXT: v_and_b32_e32 v38, v10, v3
-; SDAG-NEXT: v_and_b32_e32 v39, v10, v2
-; SDAG-NEXT: v_and_b32_e32 v10, 1, v10
+; SDAG-NEXT: v_and_b32_e32 v20, v8, v29
+; SDAG-NEXT: v_and_b32_e32 v21, v8, v28
+; SDAG-NEXT: v_and_b32_e32 v38, v8, v3
+; SDAG-NEXT: v_and_b32_e32 v39, v8, v2
+; SDAG-NEXT: v_and_b32_e32 v8, 1, v8
; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v18, v21
; SDAG-NEXT: v_subb_u32_e32 v19, vcc, v19, v20, vcc
; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v22, v39, vcc
; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v38, vcc
; SDAG-NEXT: v_or_b32_e32 v0, v16, v0
-; SDAG-NEXT: v_mov_b32_e32 v21, v11
-; SDAG-NEXT: v_mov_b32_e32 v20, v10
+; SDAG-NEXT: v_mov_b32_e32 v21, v9
+; SDAG-NEXT: v_mov_b32_e32 v20, v8
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_cbranch_execnz .LBB0_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
@@ -188,11 +188,11 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: .LBB0_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v9
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[8:9], 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v11
+; SDAG-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
; SDAG-NEXT: v_or_b32_e32 v0, v0, v16
-; SDAG-NEXT: v_or_b32_e32 v20, v11, v3
-; SDAG-NEXT: v_or_b32_e32 v21, v10, v2
+; SDAG-NEXT: v_or_b32_e32 v20, v9, v3
+; SDAG-NEXT: v_or_b32_e32 v21, v8, v2
; SDAG-NEXT: .LBB0_6: ; %Flow16
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: v_ashrrev_i32_e32 v18, 31, v7
@@ -981,14 +981,14 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v20, v20, v22, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v0, v20, v0, s[4:5]
; SDAG-NEXT: v_lshr_b64 v[16:17], v[16:17], v26
-; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v23, 0, v17, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v22, 0, v16, vcc
; SDAG-NEXT: v_add_i32_e32 v30, vcc, -1, v8
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v9, vcc
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v10, vcc
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v11, vcc
-; SDAG-NEXT: v_mov_b32_e32 v22, 0
-; SDAG-NEXT: v_mov_b32_e32 v23, 0
+; SDAG-NEXT: v_mov_b32_e32 v16, 0
+; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_mov_b32_e32 v24, 0
@@ -999,25 +999,25 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
; SDAG-NEXT: v_or_b32_e32 v19, v25, v19
; SDAG-NEXT: v_or_b32_e32 v18, v24, v18
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
+; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v1
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v20
-; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v3
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v20
-; SDAG-NEXT: v_sub_i32_e32 v20, vcc, v30, v0
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v31, v1, vcc
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v32, v16, vcc
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v33, v17, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v20, 31, v20
-; SDAG-NEXT: v_and_b32_e32 v24, v20, v8
-; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v24
-; SDAG-NEXT: v_and_b32_e32 v24, v20, v9
-; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v24, vcc
-; SDAG-NEXT: v_and_b32_e32 v24, v20, v10
-; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v16, v24, vcc
-; SDAG-NEXT: v_and_b32_e32 v24, v20, v11
-; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v17, v24, vcc
+; SDAG-NEXT: v_or_b32_e32 v20, v22, v20
+; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v22
+; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v30, v0
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v31, v1, vcc
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v32, v20, vcc
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v33, v23, vcc
+; SDAG-NEXT: v_ashrrev_i32_e32 v35, 31, v22
+; SDAG-NEXT: v_and_b32_e32 v22, v35, v8
+; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v22
+; SDAG-NEXT: v_and_b32_e32 v22, v35, v9
+; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v22, vcc
+; SDAG-NEXT: v_and_b32_e32 v22, v35, v10
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v20, v22, vcc
+; SDAG-NEXT: v_and_b32_e32 v20, v35, v11
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v20, vcc
; SDAG-NEXT: v_add_i32_e32 v26, vcc, -1, v26
; SDAG-NEXT: v_addc_u32_e32 v27, vcc, -1, v27, vcc
; SDAG-NEXT: v_addc_u32_e32 v28, vcc, -1, v28, vcc
@@ -1025,12 +1025,12 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v24, v26, v28
; SDAG-NEXT: v_or_b32_e32 v25, v27, v29
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[24:25]
-; SDAG-NEXT: v_and_b32_e32 v20, 1, v20
+; SDAG-NEXT: v_and_b32_e32 v20, 1, v35
; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
; SDAG-NEXT: v_or_b32_e32 v2, v2, v34
-; SDAG-NEXT: v_or_b32_e32 v3, v23, v3
+; SDAG-NEXT: v_or_b32_e32 v3, v17, v3
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v2, v22, v2
+; SDAG-NEXT: v_or_b32_e32 v2, v16, v2
; SDAG-NEXT: v_mov_b32_e32 v25, v21
; SDAG-NEXT: v_mov_b32_e32 v24, v20
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index b346d5ddb79aa..a653eaa936e87 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -1145,23 +1145,22 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
; CI-LABEL: write2_sgemm_sequence:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x4
+; CI-NEXT: s_lshl_b32 s2, s8, 2
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_load_dword s0, s[0:1], 0x0
-; CI-NEXT: s_lshl_b32 s1, s8, 2
-; CI-NEXT: s_add_i32 s2, s1, 0xc20
-; CI-NEXT: s_addk_i32 s1, 0xc60
-; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: s_add_i32 s1, s2, 0xc20
+; CI-NEXT: s_addk_i32 s2, 0xc60
+; CI-NEXT: v_mov_b32_e32 v0, s1
+; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: v_mov_b32_e32 v3, s0
-; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
-; CI-NEXT: v_mov_b32_e32 v0, s1
-; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
+; CI-NEXT: ds_write2_b32 v0, v3, v3 offset1:1
+; CI-NEXT: ds_write2_b32 v2, v3, v3 offset1:1
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v1
-; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
-; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:32 offset1:33
-; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:64 offset1:65
+; CI-NEXT: ds_write2_b32 v0, v3, v3 offset1:1
+; CI-NEXT: ds_write2_b32 v0, v3, v3 offset0:32 offset1:33
+; CI-NEXT: ds_write2_b32 v0, v3, v3 offset0:64 offset1:65
; CI-NEXT: s_endpgm
;
; GFX9-LABEL: write2_sgemm_sequence:
@@ -1176,13 +1175,12 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
-; GFX9-NEXT: ds_write2_b32 v2, v3, v4 offset1:1
+; GFX9-NEXT: ds_write2_b32 v0, v3, v3 offset1:1
+; GFX9-NEXT: ds_write2_b32 v2, v3, v3 offset1:1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v1
-; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
-; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:32 offset1:33
-; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:64 offset1:65
+; GFX9-NEXT: ds_write2_b32 v0, v3, v3 offset1:1
+; GFX9-NEXT: ds_write2_b32 v0, v3, v3 offset0:32 offset1:33
+; GFX9-NEXT: ds_write2_b32 v0, v3, v3 offset0:64 offset1:65
; GFX9-NEXT: s_endpgm
;
; GFX1250-LABEL: write2_sgemm_sequence:
@@ -1208,14 +1206,14 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
; GFX1250-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_lshrrev_b32 v0, 8, v0
; GFX1250-NEXT: s_addk_co_i32 s1, 0xc60
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
-; GFX1250-NEXT: ds_store_2addr_b32 v1, v2, v3 offset1:1
-; GFX1250-NEXT: ds_store_2addr_b32 v4, v2, v3 offset1:1
-; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v3 offset1:1
-; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v3 offset0:32 offset1:33
-; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v3 offset0:64 offset1:65
+; GFX1250-NEXT: ds_store_2addr_b32 v1, v2, v2 offset1:1
+; GFX1250-NEXT: ds_store_2addr_b32 v3, v2, v2 offset1:1
+; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v2 offset1:1
+; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v2 offset0:32 offset1:33
+; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v2 offset0:64 offset1:65
; GFX1250-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1
%y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 3ba9d5ff7369f..fbb2574dd7a5b 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -501,62 +501,61 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[4:5] offset:8
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e64 v4, |v1.l|
+; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX11-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] offset:8
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e64 v2, |v0.l|
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e64 v1, |v0.h|
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v2, v1
; GFX11-TRUE16-NEXT: s_cbranch_vccz .LBB0_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %frem.else
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v2, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_3
; GFX11-TRUE16-NEXT: s_branch .LBB0_8
; GFX11-TRUE16-NEXT: .LBB0_2:
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
; GFX11-TRUE16-NEXT: .LBB0_3: ; %frem.compute
-; GFX11-TRUE16-NEXT: v_frexp_mant_f32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v5, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, v3, 11
-; GFX11-TRUE16-NEXT: v_frexp_mant_f32_e32 v3, v2
-; GFX11-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v2, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v3, 1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
+; GFX11-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v4, v2
+; GFX11-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v4
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v2, 11
+; GFX11-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, v1
+; GFX11-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v2, v2, 1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, -1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_div_scale_f32 v7, null, v3, v3, 1.0
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_div_scale_f32 v6, null, v2, v2, 1.0
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_rcp_f32_e32 v8, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v5
-; GFX11-TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, 1.0, v3, 1.0
+; GFX11-TRUE16-NEXT: v_rcp_f32_e32 v7, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v2, 1.0
; GFX11-TRUE16-NEXT: s_denorm_mode 15
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_fma_f32 v9, -v7, v8, 1.0
+; GFX11-TRUE16-NEXT: v_fma_f32 v8, -v6, v7, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v8, v9, v8
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v9, v5, v8
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v7
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v8, v4, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_f32 v10, -v7, v9, v5
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v9, v10, v8
+; GFX11-TRUE16-NEXT: v_fma_f32 v9, -v6, v8, v4
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v8, v9, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_f32 v5, -v7, v9, v5
+; GFX11-TRUE16-NEXT: v_fma_f32 v4, -v6, v8, v4
; GFX11-TRUE16-NEXT: s_denorm_mode 12
-; GFX11-TRUE16-NEXT: v_div_fmas_f32 v5, v5, v8, v9
-; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v6
+; GFX11-TRUE16-NEXT: v_div_fmas_f32 v4, v4, v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_div_fixup_f32 v5, v5, v3, 1.0
+; GFX11-TRUE16-NEXT: v_div_fixup_f32 v4, v4, v2, 1.0
; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB0_7
; GFX11-TRUE16-NEXT: ; %bb.4: ; %frem.loop_body.preheader
; GFX11-TRUE16-NEXT: s_sub_i32 s2, s2, s3
@@ -565,49 +564,48 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX11-TRUE16-NEXT: .LBB0_5: ; %frem.loop_body
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, -11
; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s2, 11
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v4, v7, v5
-; GFX11-TRUE16-NEXT: v_rndne_f32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v3, v6, v4
+; GFX11-TRUE16-NEXT: v_rndne_f32_e32 v3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_f32 v4, -v4, v3, v7
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v4, v3
+; GFX11-TRUE16-NEXT: v_fma_f32 v3, -v3, v2, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, v4, 11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v3, 11
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB0_5
; GFX11-TRUE16-NEXT: ; %bb.6: ; %Flow
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v6
; GFX11-TRUE16-NEXT: .LBB0_7: ; %frem.loop_exit
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, -10, v6
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, v4, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, -10, v5
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v3, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX11-TRUE16-NEXT: v_rndne_f32_e32 v5, v5
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v4, v3, v4
+; GFX11-TRUE16-NEXT: v_rndne_f32_e32 v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_f32 v4, -v5, v3, v4
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX11-TRUE16-NEXT: v_fma_f32 v3, -v4, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v2, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v2.l, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0x7fff, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v1, v2, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v1
+; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0x7fff, v1, v0
; GFX11-TRUE16-NEXT: .LBB0_8: ; %Flow19
-; GFX11-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e64 s2, 0x7c00, |v1.l|
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v0.h
+; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e64 s2, 0x7c00, |v0.l|
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v3.l, s2
-; GFX11-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: frem_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/image-sample-waterfall.ll b/llvm/test/CodeGen/AMDGPU/image-sample-waterfall.ll
index 3206e9593320e..0189aa8f155d2 100644
--- a/llvm/test/CodeGen/AMDGPU/image-sample-waterfall.ll
+++ b/llvm/test/CodeGen/AMDGPU/image-sample-waterfall.ll
@@ -24,8 +24,11 @@ declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <
; GCN-NEXT: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[AND]]
; GCN-NEXT: s_nop 0
; GCN-NEXT: image_gather4 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, s[[[SREG0]]:[[SREG7]]], {{s\[[0-9]+:[0-9]+\]}} dmask:0x1
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GCN-NEXT: ; implicit-def: $vgpr10
; GCN-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GCN-NEXT: ; implicit-def: $vgpr12_vgpr13
+; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN-NEXT: ; implicit-def: $vgpr6_vgpr7
; GCN-NEXT: s_xor_b64 exec, exec, [[SAVE]]
; GCN-NEXT: s_cbranch_execnz [[RSRC_LOOP]]
define amdgpu_ps <4 x float> @water_loop_rsrc(<8 x i32> %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
@@ -50,8 +53,9 @@ main_body:
; GCN-NEXT: s_nop 0
; GCN-NEXT: image_gather4 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, s[[[SREG0]]:[[SREG3]]] dmask:0x1
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GCN-NEXT: ; implicit-def: $vgpr8
; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN-NEXT: ; implicit-def: $vgpr6_vgpr7
; GCN-NEXT: s_xor_b64 exec, exec, [[SAVE]]
; GCN-NEXT: s_cbranch_execnz [[SAMP_LOOP]]
define amdgpu_ps <4 x float> @water_loop_samp(<8 x i32> inreg %rsrc, <4 x i32> %samp, float %s, float %t) {
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index ef51ec106a935..e2cc54fe53e46 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -169,7 +169,7 @@ define void @test_indirect_call_vgpr_ptr(ptr %fptr) #0 {
; GCN-NEXT: s_mov_b32 s14, s51
; GCN-NEXT: s_mov_b32 s15, s50
; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: ; implicit-def: $vgpr31
; GCN-NEXT: s_xor_b64 exec, exec, s[64:65]
; GCN-NEXT: s_cbranch_execnz .LBB2_1
@@ -343,7 +343,7 @@ define void @test_indirect_call_vgpr_ptr_arg(ptr %fptr) #0 {
; GCN-NEXT: s_mov_b32 s15, s50
; GCN-NEXT: v_mov_b32_e32 v0, v2
; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: ; implicit-def: $vgpr31
; GCN-NEXT: ; implicit-def: $vgpr2
; GCN-NEXT: s_xor_b64 exec, exec, s[64:65]
@@ -517,14 +517,14 @@ define i32 @test_indirect_call_vgpr_ptr_ret(ptr %fptr) #0 {
; GCN-NEXT: s_mov_b32 s14, s51
; GCN-NEXT: s_mov_b32 s15, s50
; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GCN-NEXT: v_mov_b32_e32 v2, v0
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: ; implicit-def: $vgpr31
; GCN-NEXT: s_xor_b64 exec, exec, s[64:65]
; GCN-NEXT: s_cbranch_execnz .LBB4_1
; GCN-NEXT: ; %bb.2:
; GCN-NEXT: s_mov_b64 exec, s[54:55]
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 1, v2
+; GCN-NEXT: v_add_i32_e32 v0, vcc, 1, v1
; GCN-NEXT: v_readlane_b32 s30, v40, 16
; GCN-NEXT: v_readlane_b32 s31, v40, 17
; GCN-NEXT: v_readlane_b32 s65, v40, 15
@@ -701,7 +701,7 @@ define void @test_indirect_call_vgpr_ptr_in_branch(ptr %fptr, i1 %cond) #0 {
; GCN-NEXT: s_mov_b32 s14, s51
; GCN-NEXT: s_mov_b32 s15, s50
; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: ; implicit-def: $vgpr31
; GCN-NEXT: s_xor_b64 exec, exec, s[66:67]
; GCN-NEXT: s_cbranch_execnz .LBB5_2
@@ -877,7 +877,7 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) #0 {
; GCN-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GCN-NEXT: s_movk_i32 s4, 0x7b
; GCN-NEXT: s_swappc_b64 s[30:31], s[8:9]
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: s_xor_b64 exec, exec, s[10:11]
; GCN-NEXT: s_cbranch_execnz .LBB6_1
; GCN-NEXT: ; %bb.2:
@@ -1009,7 +1009,7 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_reuse(i32 %i, ptr %fptr) #0 {
; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GCN-NEXT: v_mov_b32_e32 v0, v40
; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
; GCN-NEXT: s_cbranch_execnz .LBB7_1
; GCN-NEXT: ; %bb.2:
@@ -1146,14 +1146,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_return(i32 %i, ptr %fptr) #0 {
; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[1:2]
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_swappc_b64 s[30:31], s[8:9]
-; GCN-NEXT: v_mov_b32_e32 v3, v0
-; GCN-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GCN-NEXT: v_mov_b32_e32 v2, v0
+; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: s_xor_b64 exec, exec, s[6:7]
; GCN-NEXT: s_cbranch_execnz .LBB8_1
; GCN-NEXT: ; %bb.2:
; GCN-NEXT: s_mov_b64 exec, s[4:5]
-; GCN-NEXT: v_mov_b32_e32 v0, v3
+; GCN-NEXT: v_mov_b32_e32 v0, v2
; GCN-NEXT: v_readlane_b32 s30, v40, 14
; GCN-NEXT: v_readlane_b32 s31, v40, 15
; GCN-NEXT: v_readlane_b32 s55, v40, 13
@@ -1279,7 +1279,7 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) #0 {
; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1]
; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
; GCN-NEXT: s_cbranch_execnz .LBB9_1
; GCN-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 37d7c24cb6869..d86972317ad69 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -1135,10 +1135,9 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
; SI-NEXT: s_or_b32 s7, s5, s4
; SI-NEXT: s_lshl_b64 s[4:5], 0xffff, s6
; SI-NEXT: v_mov_b32_e32 v4, s7
-; SI-NEXT: v_mov_b32_e32 v5, s7
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_bfi_b32 v3, s5, v4, v3
-; SI-NEXT: v_bfi_b32 v2, s4, v5, v2
+; SI-NEXT: v_bfi_b32 v2, s4, v4, v2
; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
; SI-NEXT: s_endpgm
;
@@ -1163,11 +1162,10 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
; VI-NEXT: s_lshl_b64 s[0:1], 0xffff, s3
; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: v_mov_b32_e32 v5, s2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_bfi_b32 v1, s1, v4, v1
-; VI-NEXT: v_bfi_b32 v0, s0, v5, v0
+; VI-NEXT: v_bfi_b32 v0, s0, v4, v0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
@@ -1182,10 +1180,9 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
; GFX900-NEXT: s_pack_ll_b32_b16 s4, s4, s4
; GFX900-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
; GFX900-NEXT: v_mov_b32_e32 v3, s4
-; GFX900-NEXT: v_mov_b32_e32 v4, s4
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_bfi_b32 v1, s3, v3, v1
-; GFX900-NEXT: v_bfi_b32 v0, s2, v4, v0
+; GFX900-NEXT: v_bfi_b32 v0, s2, v3, v0
; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX900-NEXT: s_endpgm
;
@@ -1201,10 +1198,9 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
; GFX942-NEXT: s_pack_ll_b32_b16 s4, s6, s6
; GFX942-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
; GFX942-NEXT: v_mov_b32_e32 v3, s4
-; GFX942-NEXT: v_mov_b32_e32 v4, s4
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_bfi_b32 v1, s3, v3, v1
-; GFX942-NEXT: v_bfi_b32 v0, s2, v4, v0
+; GFX942-NEXT: v_bfi_b32 v0, s2, v3, v0
; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index c24ee53de2e76..90d4ef1f50898 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -2430,10 +2430,9 @@ define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(ptr addrspace(1) %
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s4
; GFX9-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_bfi_b32 v1, s3, v3, v1
-; GFX9-NEXT: v_bfi_b32 v0, s2, v4, v0
+; GFX9-NEXT: v_bfi_b32 v0, s2, v3, v0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -2458,11 +2457,10 @@ define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(ptr addrspace(1) %
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
; VI-NEXT: s_lshl_b64 s[0:1], 0xffff, s3
; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: v_mov_b32_e32 v5, s2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_bfi_b32 v1, s1, v4, v1
-; VI-NEXT: v_bfi_b32 v0, s0, v5, v0
+; VI-NEXT: v_bfi_b32 v0, s0, v4, v0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
@@ -2487,11 +2485,10 @@ define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(ptr addrspace(1) %
; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v2
; CI-NEXT: s_lshl_b64 s[0:1], 0xffff, s3
; CI-NEXT: v_mov_b32_e32 v4, s2
-; CI-NEXT: v_mov_b32_e32 v5, s2
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_bfi_b32 v1, s1, v4, v1
-; CI-NEXT: v_bfi_b32 v0, s0, v5, v0
+; CI-NEXT: v_bfi_b32 v0, s0, v4, v0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
index 373648ace8621..9f8d66b955682 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
@@ -146,10 +146,10 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_ssssss(i64 inreg %node_p
define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, i32 %offset, <4 x i32> %tdescr, ptr addrspace(1) %origin, ptr addrspace(1) %dir) {
; GFX12-SDAG-LABEL: image_bvh8_intersect_ray_vvvvvv:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v28, v9 :: v_dual_mov_b32 v19, v7
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v20, v8 :: v_dual_mov_b32 v23, v5
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v18, v6 :: v_dual_mov_b32 v21, v3
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v22, v4 :: v_dual_mov_b32 v25, v1
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v28, v9 :: v_dual_mov_b32 v23, v8
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v21, v6
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v19, v4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v18, v3 :: v_dual_mov_b32 v25, v1
; GFX12-SDAG-NEXT: v_dual_mov_b32 v26, v2 :: v_dual_mov_b32 v27, 0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v24, v0
; GFX12-SDAG-NEXT: s_mov_b32 s4, exec_lo
@@ -165,18 +165,19 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, fl
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[0:3]
+; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[18:20], v[21:23], v28], s[0:3]
; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr10
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr24_vgpr25
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr26_vgpr27
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr28
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr12_vgpr13
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b96 v[14:15], v[21:23], off
-; GFX12-SDAG-NEXT: global_store_b96 v[16:17], v[18:20], off
+; GFX12-SDAG-NEXT: global_store_b96 v[14:15], v[18:20], off
+; GFX12-SDAG-NEXT: global_store_b96 v[16:17], v[21:23], off
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: image_bvh8_intersect_ray_vvvvvv:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index 195e72ecaf336..7781b7443d52e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -152,9 +152,9 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-LABEL: image_bvh_dual_intersect_ray_vvvvvv:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: v_dual_mov_b32 v26, v10 :: v_dual_mov_b32 v25, v9
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v21, v8 :: v_dual_mov_b32 v20, v7
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v19, v6 :: v_dual_mov_b32 v24, v5
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v23, v4 :: v_dual_mov_b32 v22, v3
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v24, v8 :: v_dual_mov_b32 v23, v7
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v22, v6 :: v_dual_mov_b32 v21, v5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v20, v4 :: v_dual_mov_b32 v19, v3
; GFX12-SDAG-NEXT: v_dual_mov_b32 v29, v2 :: v_dual_mov_b32 v28, v1
; GFX12-SDAG-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v30, 0
; GFX12-SDAG-NEXT: s_mov_b32 s4, exec_lo
@@ -170,18 +170,19 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[11:12]
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[13:14]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[0:3]
+; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[19:21], v[22:24], v[25:26]], s[0:3]
; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr11_vgpr12_vgpr13_vgpr14
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr11
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr27_vgpr28
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr29_vgpr30
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr25_vgpr26
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr13_vgpr14
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b96 v[15:16], v[22:24], off
-; GFX12-SDAG-NEXT: global_store_b96 v[17:18], v[19:21], off
+; GFX12-SDAG-NEXT: global_store_b96 v[15:16], v[19:21], off
+; GFX12-SDAG-NEXT: global_store_b96 v[17:18], v[22:24], off
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: image_bvh_dual_intersect_ray_vvvvvv:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index e035bfbf18fb5..565b96971ef90 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -24,8 +24,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -51,8 +52,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr4
+; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-NEXT: ; %bb.2:
@@ -77,8 +79,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: ; implicit-def: $vgpr4
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
; GFX8-NEXT: ; %bb.2:
@@ -105,8 +108,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
@@ -131,8 +135,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
@@ -158,8 +163,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
@@ -186,8 +192,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-FAKE16-NEXT: ; %bb.2:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
index 9f30255a07095..f69fc3c3672a4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -20,59 +20,32 @@ define amdgpu_ps void @tensor_load_to_lds_d4(<4 x i32> inreg %D0, <8 x i32> inre
}
define amdgpu_ps void @tensor_load_to_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
-; GFX1250-SDAG-LABEL: tensor_load_to_lds_d4_vector:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s12, v12
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s13, v13
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s14, v14
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s15, v15
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s16, v16
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s17, v17
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s18, v18
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s19, v19
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
-; GFX1250-SDAG-NEXT: s_endpgm
-;
-; GFX1250-GISEL-LABEL: tensor_load_to_lds_d4_vector:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s12, v12
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s13, v13
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s14, v14
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s15, v15
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s16, v16
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s17, v17
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s18, v18
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s19, v19
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
-; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-LABEL: tensor_load_to_lds_d4_vector:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1250-NEXT: v_readfirstlane_b32 s10, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
+; GFX1250-NEXT: v_readfirstlane_b32 s12, v12
+; GFX1250-NEXT: v_readfirstlane_b32 s13, v13
+; GFX1250-NEXT: v_readfirstlane_b32 s14, v14
+; GFX1250-NEXT: v_readfirstlane_b32 s15, v15
+; GFX1250-NEXT: v_readfirstlane_b32 s16, v16
+; GFX1250-NEXT: v_readfirstlane_b32 s17, v17
+; GFX1250-NEXT: v_readfirstlane_b32 s18, v18
+; GFX1250-NEXT: v_readfirstlane_b32 s19, v19
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
+; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> zeroinitializer, i32 0)
ret void
}
@@ -88,43 +61,24 @@ define amdgpu_ps void @tensor_load_to_lds_d2(<4 x i32> inreg %D0, <8 x i32> inre
}
define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
-; GFX1250-SDAG-LABEL: tensor_load_to_lds_d2_vector:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
-; GFX1250-SDAG-NEXT: s_endpgm
-;
-; GFX1250-GISEL-LABEL: tensor_load_to_lds_d2_vector:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
-; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-LABEL: tensor_load_to_lds_d2_vector:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1250-NEXT: v_readfirstlane_b32 s10, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 27)
ret void
}
@@ -140,59 +94,32 @@ define amdgpu_ps void @tensor_store_from_lds_d4(<4 x i32> inreg %D0, <8 x i32> i
}
define amdgpu_ps void @tensor_store_from_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
-; GFX1250-SDAG-LABEL: tensor_store_from_lds_d4_vector:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s12, v12
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s13, v13
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s14, v14
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s15, v15
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s16, v16
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s17, v17
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s18, v18
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s19, v19
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
-; GFX1250-SDAG-NEXT: s_endpgm
-;
-; GFX1250-GISEL-LABEL: tensor_store_from_lds_d4_vector:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s12, v12
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s13, v13
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s14, v14
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s15, v15
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s16, v16
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s17, v17
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s18, v18
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s19, v19
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-LABEL: tensor_store_from_lds_d4_vector:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1250-NEXT: v_readfirstlane_b32 s10, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
+; GFX1250-NEXT: v_readfirstlane_b32 s12, v12
+; GFX1250-NEXT: v_readfirstlane_b32 s13, v13
+; GFX1250-NEXT: v_readfirstlane_b32 s14, v14
+; GFX1250-NEXT: v_readfirstlane_b32 s15, v15
+; GFX1250-NEXT: v_readfirstlane_b32 s16, v16
+; GFX1250-NEXT: v_readfirstlane_b32 s17, v17
+; GFX1250-NEXT: v_readfirstlane_b32 s18, v18
+; GFX1250-NEXT: v_readfirstlane_b32 s19, v19
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> zeroinitializer, i32 22)
ret void
}
@@ -208,43 +135,24 @@ define amdgpu_ps void @tensor_store_from_lds_d2(<4 x i32> inreg %D0, <8 x i32> i
}
define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
-; GFX1250-SDAG-LABEL: tensor_store_from_lds_d2_vector:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: tensor_store_from_lds s[8:11], s[0:7]
-; GFX1250-SDAG-NEXT: s_endpgm
-;
-; GFX1250-GISEL-LABEL: tensor_store_from_lds_d2_vector:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v6
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: tensor_store_from_lds s[8:11], s[0:7]
-; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-LABEL: tensor_store_from_lds_d2_vector:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1250-NEXT: v_readfirstlane_b32 s10, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: tensor_store_from_lds s[8:11], s[0:7]
+; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 01bdaacbabac2..8e65676b7c22f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -5491,13 +5491,11 @@ define amdgpu_kernel void @constant_sextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u8 v0, v4, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b32_e32 v2, 1, v0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v1, 1, v0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-NEXT: v_bfe_i32 v2, v2, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bfe_i32 v2, v1, 0, 1
+; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v3, 31, v2
; GFX1250-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
@@ -5765,21 +5763,21 @@ define amdgpu_kernel void @constant_sextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v5, 0
+; GFX1250-NEXT: v_mov_b32_e32 v6, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u8 v0, v5, s[2:3] nv
+; GFX1250-NEXT: global_load_u8 v0, v6, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 2, v0 :: v_dual_lshrrev_b32 v4, 1, v0
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 2, v0 :: v_dual_lshrrev_b32 v2, 1, v0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
-; GFX1250-NEXT: v_bfe_i32 v2, v4, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v7, 31, v6
-; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX1250-NEXT: v_bfe_i32 v4, v1, 0, 1
+; GFX1250-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX1250-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_ashrrev_i32 v3, 31, v2
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b64 v5, v[6:7], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX1250-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v6, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
%ext = sext <3 x i1> %load to <3 x i64>
@@ -6080,25 +6078,25 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v9, 0
+; GFX1250-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u8 v0, v9, s[2:3] nv
+; GFX1250-NEXT: global_load_u8 v0, v8, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 3, v0 :: v_dual_lshrrev_b32 v4, 2, v0
-; GFX1250-NEXT: v_lshrrev_b32_e32 v8, 1, v0
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 3, v0 :: v_dual_lshrrev_b32 v2, 2, v0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v3, 1, v0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
-; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 1
+; GFX1250-NEXT: v_bfe_i32 v6, v1, 0, 1
+; GFX1250-NEXT: v_bfe_i32 v4, v2, 0, 1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-NEXT: v_bfe_i32 v2, v8, 0, 1
+; GFX1250-NEXT: v_bfe_i32 v2, v3, 0, 1
; GFX1250-NEXT: v_dual_ashrrev_i32 v7, 31, v6 :: v_dual_ashrrev_i32 v5, 31, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX1250-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
%ext = sext <4 x i1> %load to <4 x i64>
@@ -6357,7 +6355,7 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX8-NEXT: s_lshr_b32 s10, s3, 2
; GFX8-NEXT: s_lshr_b32 s12, s3, 3
; GFX8-NEXT: s_lshr_b32 s14, s3, 1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_bfe_i32 v0, s3, 0, 1
; GFX8-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
; GFX8-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX8-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
@@ -6386,7 +6384,6 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 1
; GFX8-NEXT: v_mov_b32_e32 v12, s10
; GFX8-NEXT: v_mov_b32_e32 v13, s11
; GFX8-NEXT: v_mov_b32_e32 v14, s12
@@ -6459,8 +6456,6 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX12-NEXT: global_load_d16_u8 v0, v16, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v9, s3
; GFX12-NEXT: s_lshr_b32 s2, s3, 6
; GFX12-NEXT: s_lshr_b32 s4, s3, 7
; GFX12-NEXT: s_lshr_b32 s6, s3, 4
@@ -6468,30 +6463,28 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX12-NEXT: s_lshr_b32 s10, s3, 2
; GFX12-NEXT: s_lshr_b32 s12, s3, 3
; GFX12-NEXT: s_lshr_b32 s14, s3, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_bfe_i32 v0, s3, 0, 1
; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: v_bfe_i32 v12, v9, 0, 1
; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX12-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
-; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
-; GFX12-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX12-NEXT: v_mov_b32_e32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX12-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v11, s9
+; GFX12-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s11
+; GFX12-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v15, s13
+; GFX12-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
+; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v8i1_to_v8i64:
@@ -6503,8 +6496,6 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250-NEXT: global_load_u8 v0, v16, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v10, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 6
; GFX1250-NEXT: s_lshr_b32 s4, s3, 7
; GFX1250-NEXT: s_lshr_b32 s6, s3, 4
@@ -6512,27 +6503,27 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_lshr_b32 s10, s3, 2
; GFX1250-NEXT: s_lshr_b32 s12, s3, 3
; GFX1250-NEXT: s_lshr_b32 s14, s3, 1
+; GFX1250-NEXT: v_bfe_i32 v0, s3, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX1250-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-NEXT: v_bfe_i32 v12, v10, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX1250-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
+; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX1250-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
-; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
-; GFX1250-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
-; GFX1250-NEXT: v_dual_ashrrev_i32 v13, 31, v12 :: v_dual_mov_b32 v15, s15
+; GFX1250-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v11, s9
+; GFX1250-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s11
+; GFX1250-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v15, s13
+; GFX1250-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX1250-NEXT: s_clause 0x3
-; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <8 x i1>, ptr addrspace(4) %in
%ext = sext <8 x i1> %load to <8 x i64>
@@ -6603,84 +6594,85 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-LABEL: constant_zextload_v16i1_to_v16i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v11, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v0
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x1000d
-; GFX8-NEXT: s_bfe_u32 s6, s2, 0x10007
-; GFX8-NEXT: s_bfe_u32 s7, s2, 0x10003
-; GFX8-NEXT: s_bfe_u32 s8, s2, 0x10001
-; GFX8-NEXT: s_and_b32 s9, s2, 1
-; GFX8-NEXT: s_bfe_u32 s10, s2, 0x10002
-; GFX8-NEXT: s_bfe_u32 s11, s2, 0x10004
-; GFX8-NEXT: s_bfe_u32 s12, s2, 0x10006
-; GFX8-NEXT: s_bfe_u32 s13, s2, 0x1000c
+; GFX8-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX8-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX8-NEXT: s_bfe_u32 s5, s2, 0x10007
+; GFX8-NEXT: s_bfe_u32 s6, s2, 0x10003
+; GFX8-NEXT: s_bfe_u32 s7, s2, 0x10001
+; GFX8-NEXT: s_and_b32 s8, s2, 1
+; GFX8-NEXT: s_bfe_u32 s9, s2, 0x10002
+; GFX8-NEXT: s_bfe_u32 s10, s2, 0x10004
+; GFX8-NEXT: s_bfe_u32 s11, s2, 0x10006
+; GFX8-NEXT: s_bfe_u32 s12, s2, 0x1000c
; GFX8-NEXT: s_bfe_u32 s2, s2, 0x1000a
-; GFX8-NEXT: v_bfe_u32 v4, v2, 11, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 15, v2
-; GFX8-NEXT: v_bfe_u32 v12, v2, 5, 1
-; GFX8-NEXT: v_bfe_u32 v6, v2, 14, 1
-; GFX8-NEXT: v_bfe_u32 v0, v2, 8, 1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: s_add_u32 s2, s0, 0x50
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v11, s3
-; GFX8-NEXT: v_mov_b32_e32 v10, s2
+; GFX8-NEXT: v_mov_b32_e32 v13, s3
+; GFX8-NEXT: v_mov_b32_e32 v12, s2
; GFX8-NEXT: s_add_u32 s2, s0, 64
+; GFX8-NEXT: v_bfe_u32 v2, v4, 11, 1
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[2:5]
-; GFX8-NEXT: v_mov_b32_e32 v11, s3
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_mov_b32_e32 v10, s2
+; GFX8-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v13, s3
+; GFX8-NEXT: v_mov_b32_e32 v12, s2
; GFX8-NEXT: s_add_u32 s2, s0, 0x70
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 15, v4
+; GFX8-NEXT: v_bfe_u32 v14, v4, 5, 1
+; GFX8-NEXT: v_bfe_u32 v8, v4, 14, 1
+; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 1
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: s_add_u32 s2, s0, 0x60
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[6:9]
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
-; GFX8-NEXT: v_mov_b32_e32 v2, s13
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: s_add_u32 s2, s0, 48
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s12
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: s_add_u32 s2, s0, 32
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s11
+; GFX8-NEXT: v_mov_b32_e32 v2, s5
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s11
-; GFX8-NEXT: v_mov_b32_e32 v4, v12
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: s_add_u32 s2, s0, 16
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_mov_b32_e32 v2, v14
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s10
-; GFX8-NEXT: v_mov_b32_e32 v4, s7
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v2, s9
-; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s9
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v2, s7
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
; EG-LABEL: constant_zextload_v16i1_to_v16i64:
@@ -6759,61 +6751,60 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-LABEL: constant_zextload_v16i1_to_v16i64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-NEXT: v_mov_b32_e32 v9, v3
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-NEXT: v_mov_b32_e32 v5, v1
+; GFX12-NEXT: v_mov_b32_e32 v9, v1
+; GFX12-NEXT: v_mov_b32_e32 v11, v1
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_d16_b16 v0, v3, s[2:3]
+; GFX12-NEXT: global_load_d16_b16 v0, v1, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v0
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-NEXT: v_bfe_u32 v0, v6, 8, 1
+; GFX12-NEXT: v_bfe_u32 v2, v4, 11, 1
+; GFX12-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX12-NEXT: v_bfe_u32 v6, v4, 5, 1
+; GFX12-NEXT: s_bfe_u32 s6, s2, 0x10002
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX12-NEXT: v_lshrrev_b32_e32 v10, 15, v4
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10001
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-NEXT: v_bfe_u32 v8, v4, 14, 1
+; GFX12-NEXT: v_bfe_u32 v4, v4, 8, 1
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, v6
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10003
; GFX12-NEXT: s_and_b32 s2, s2, 1
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX12-NEXT: v_mov_b32_e32 v6, s3
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
+; GFX12-NEXT: v_mov_b32_e32 v0, s6
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s5
+; GFX12-NEXT: s_clause 0x2
+; GFX12-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:112
+; GFX12-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:64
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
@@ -6864,58 +6855,57 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v7, 0
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v5, v1
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u16 v0, v3, s[2:3] nv
+; GFX1250-REAL16-NEXT: global_load_u16 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000a
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s5
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10009
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-REAL16-NEXT: v_and_b32_e64 v4, s2, 1
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s4
; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s4
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s4
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v2, s4
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10002
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v6, s4
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-REAL16-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: s_clause 0x1
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX1250-REAL16-NEXT: global_store_b128 v1, v[4:7], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
@@ -7024,7 +7014,7 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-NEXT: s_lshr_b32 s26, s3, 2
; GFX8-NEXT: s_lshr_b32 s28, s3, 3
; GFX8-NEXT: s_lshr_b32 s30, s3, 1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_bfe_i32 v0, s3, 0, 1
; GFX8-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
; GFX8-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX8-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
@@ -7097,7 +7087,6 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[10:13]
; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 1
; GFX8-NEXT: v_mov_b32_e32 v14, s26
; GFX8-NEXT: v_mov_b32_e32 v15, s27
; GFX8-NEXT: v_mov_b32_e32 v16, s28
@@ -7207,14 +7196,12 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: global_load_d16_b16 v0, v32, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v28, s3
; GFX12-NEXT: s_lshr_b32 s2, s3, 14
; GFX12-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-NEXT: s_lshr_b32 s6, s3, 12
; GFX12-NEXT: s_lshr_b32 s8, s3, 13
; GFX12-NEXT: s_lshr_b32 s10, s3, 10
; GFX12-NEXT: s_lshr_b32 s12, s3, 11
-; GFX12-NEXT: s_lshr_b32 s6, s3, 12
; GFX12-NEXT: s_lshr_b32 s14, s3, 8
; GFX12-NEXT: s_lshr_b32 s16, s3, 9
; GFX12-NEXT: s_lshr_b32 s18, s3, 6
@@ -7224,51 +7211,48 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_lshr_b32 s26, s3, 2
; GFX12-NEXT: s_lshr_b32 s28, s3, 3
; GFX12-NEXT: s_lshr_b32 s30, s3, 1
+; GFX12-NEXT: v_bfe_i32 v0, s3, 0, 1
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX12-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX12-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v11, s9
; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
-; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
-; GFX12-NEXT: v_mov_b32_e32 v13, s15
-; GFX12-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
-; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
-; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
+; GFX12-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s11
+; GFX12-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v15, s13
+; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
-; GFX12-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
-; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
-; GFX12-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
-; GFX12-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
-; GFX12-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
-; GFX12-NEXT: v_mov_b32_e32 v30, s30
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-NEXT: s_clause 0x5
-; GFX12-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v17, s15
+; GFX12-NEXT: v_dual_mov_b32 v18, s16 :: v_dual_mov_b32 v19, s17
+; GFX12-NEXT: v_dual_mov_b32 v20, s18 :: v_dual_mov_b32 v21, s19
+; GFX12-NEXT: v_dual_mov_b32 v22, s20 :: v_dual_mov_b32 v23, s21
+; GFX12-NEXT: v_dual_mov_b32 v24, s22 :: v_dual_mov_b32 v25, s23
+; GFX12-NEXT: v_dual_mov_b32 v26, s24 :: v_dual_mov_b32 v27, s25
+; GFX12-NEXT: v_dual_mov_b32 v28, s26 :: v_dual_mov_b32 v29, s27
+; GFX12-NEXT: v_dual_mov_b32 v30, s28 :: v_dual_mov_b32 v31, s29
+; GFX12-NEXT: v_dual_mov_b32 v2, s30 :: v_dual_mov_b32 v3, s31
+; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-NEXT: s_clause 0x7
+; GFX12-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:112
+; GFX12-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:96
+; GFX12-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:80
+; GFX12-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX12-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v32, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
@@ -7280,14 +7264,12 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v28, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
; GFX1250-NEXT: s_lshr_b32 s8, s3, 13
; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
; GFX1250-NEXT: s_lshr_b32 s12, s3, 11
-; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
; GFX1250-NEXT: s_lshr_b32 s14, s3, 8
; GFX1250-NEXT: s_lshr_b32 s16, s3, 9
; GFX1250-NEXT: s_lshr_b32 s18, s3, 6
@@ -7297,47 +7279,47 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_lshr_b32 s26, s3, 2
; GFX1250-NEXT: s_lshr_b32 s28, s3, 3
; GFX1250-NEXT: s_lshr_b32 s30, s3, 1
-; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: v_bfe_i32 v0, s3, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v11, s9
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
-; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
-; GFX1250-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
-; GFX1250-NEXT: v_mov_b32_e32 v12, s14
-; GFX1250-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX1250-NEXT: v_dual_mov_b32 v13, s15 :: v_dual_mov_b32 v14, s16
-; GFX1250-NEXT: v_dual_mov_b32 v15, s17 :: v_dual_mov_b32 v16, s18
+; GFX1250-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s11
+; GFX1250-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v15, s13
+; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v17, s19 :: v_dual_mov_b32 v18, s20
-; GFX1250-NEXT: v_dual_mov_b32 v19, s21 :: v_dual_mov_b32 v20, s22
-; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v21, s23 :: v_dual_mov_b32 v22, s24
-; GFX1250-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v24, s26
-; GFX1250-NEXT: v_dual_mov_b32 v25, s27 :: v_dual_mov_b32 v26, s28
-; GFX1250-NEXT: v_dual_mov_b32 v27, s29 :: v_dual_mov_b32 v30, s30
-; GFX1250-NEXT: v_dual_mov_b32 v31, s31 :: v_dual_ashrrev_i32 v29, 31, v28
+; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v17, s15
+; GFX1250-NEXT: v_dual_mov_b32 v18, s16 :: v_dual_mov_b32 v19, s17
+; GFX1250-NEXT: v_dual_mov_b32 v20, s18 :: v_dual_mov_b32 v21, s19
+; GFX1250-NEXT: v_dual_mov_b32 v22, s20 :: v_dual_mov_b32 v23, s21
+; GFX1250-NEXT: v_dual_mov_b32 v24, s22 :: v_dual_mov_b32 v25, s23
+; GFX1250-NEXT: v_dual_mov_b32 v26, s24 :: v_dual_mov_b32 v27, s25
+; GFX1250-NEXT: v_dual_mov_b32 v28, s26 :: v_dual_mov_b32 v29, s27
+; GFX1250-NEXT: v_dual_mov_b32 v30, s28 :: v_dual_mov_b32 v31, s29
+; GFX1250-NEXT: v_dual_mov_b32 v2, s30 :: v_dual_mov_b32 v3, s31
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX1250-NEXT: s_clause 0x7
-; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:96
+; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = sext <16 x i1> %load to <16 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 73e0f6d4f1cf3..caf9268c82c7a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -5837,17 +5837,17 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i64(ptr addrspace(1) %out,
;
; GCN-NOHSA-VI-LABEL: global_sextload_v4i16_to_v4i64:
; GCN-NOHSA-VI: ; %bb.0:
-; GCN-NOHSA-VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s10, s2
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s7
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s3
; GCN-NOHSA-VI-NEXT: buffer_load_dwordx2 v[1:2], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s5, s1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
@@ -5859,8 +5859,8 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i64(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
; EG-LABEL: global_sextload_v4i16_to_v4i64:
@@ -6996,60 +6996,60 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[1:4], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[5:8], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v8, v0, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v3, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v3, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v1, v6, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v9, v2, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v2, 0, 16
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v4, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v14, 16, v4
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v25, v5, 0, 16
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v2, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v5, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v1, 0, 16
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v27, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v7, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v10, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v11, v11, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v12, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v14, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v23, v3, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v8, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v26, 31, v25
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v28, 31, v27
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v22, 31, v21
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v26, v1, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v16, v3, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v6
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v20, v6, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v4, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v1, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v9, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v11, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v18, v3, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v5, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v7, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v24, 31, v23
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v21, 31, v20
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 31, v24
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v27, 31, v26
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
; EG-LABEL: global_sextload_v16i16_to_v16i64:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index 2c79134f785bc..87938bc0eae27 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -5741,17 +5741,17 @@ define amdgpu_kernel void @global_sextload_v4i8_to_v4i64(ptr addrspace(1) %out,
;
; GCN-NOHSA-VI-LABEL: global_sextload_v4i8_to_v4i64:
; GCN-NOHSA-VI: ; %bb.0:
-; GCN-NOHSA-VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s10, s2
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s7
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s3
; GCN-NOHSA-VI-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s5, s1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v4, 24, v0
@@ -5764,8 +5764,8 @@ define amdgpu_kernel void @global_sextload_v4i8_to_v4i64(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
; EG-LABEL: global_sextload_v4i8_to_v4i64:
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index fd5710edb1257..734bcfdd4378a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -7152,120 +7152,120 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
-; VI-DS128-NEXT: ds_read_b128 v[3:6], v0
-; VI-DS128-NEXT: ds_read_b128 v[7:10], v0 offset:16
+; VI-DS128-NEXT: v_mov_b32_e32 v4, s1
+; VI-DS128-NEXT: ds_read_b128 v[0:3], v4
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v4 offset:16
+; VI-DS128-NEXT: v_mov_b32_e32 v16, s0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; VI-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:80
-; VI-DS128-NEXT: v_bfe_i32 v11, v7, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:64
-; VI-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:112
+; VI-DS128-NEXT: v_bfe_i32 v12, v5, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; VI-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v10, v0, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:80
; VI-DS128-NEXT: v_bfe_i32 v12, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; VI-DS128-NEXT: v_bfe_i32 v14, v9, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
-; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v7
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:64
+; VI-DS128-NEXT: v_bfe_i32 v12, v7, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:112
+; VI-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:96
+; VI-DS128-NEXT: v_bfe_i32 v4, v2, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v12, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
-; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
-; VI-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3]
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: ds_write_b128 v16, v[0:3] offset:48
+; VI-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:32
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:16
+; VI-DS128-NEXT: ds_write_b128 v16, v[8:11]
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_sextload_v16i16_to_v16i64:
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[3:6], v0
-; GFX9-DS128-NEXT: ds_read_b128 v[7:10], v0 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v4
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v4 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v16, s0
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_mov_b32_e32 v8, s0
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:80
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v7, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:64
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:112
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v5, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v0, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:80
; GFX9-DS128-NEXT: v_bfe_i32 v12, v4, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v9, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v7
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:64
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v7, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:112
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GFX9-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:96
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v1, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3]
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[0:3] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[8:11]
; GFX9-DS128-NEXT: s_endpgm
%load = load <16 x i16>, ptr addrspace(3) %in
%ext = sext <16 x i16> %load to <16 x i64>
@@ -8793,20 +8793,20 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v13, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v13 offset:48
-; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v13 offset:32
+; GFX9-DS128-NEXT: v_mov_b32_e32 v17, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v17 offset:48
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v17 offset:32
; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v13
-; GFX9-DS128-NEXT: ds_read_b128 v[18:21], v13 offset:16
+; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v17
+; GFX9-DS128-NEXT: ds_read_b128 v[17:20], v17 offset:16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v6, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v6, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[14:17] offset:224
+; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:224
; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
@@ -8852,30 +8852,30 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v20
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v19
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:144
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v20, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v19, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v21
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v20
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v21, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v20, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v17
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:112
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v17, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v19
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:64
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v19, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 04d763f0ea6ea..38352700442c3 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -16,6 +16,7 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX9_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX9_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -24,15 +25,16 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v4, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
; GFX9_W64-NEXT: ; implicit-def: $vgpr4
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX9_W64-NEXT: ; %bb.2:
; GFX9_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX9_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX9_W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1010_W32-LABEL: mubuf_vgpr:
@@ -42,22 +44,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v4, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
+; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v1
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
; GFX1010_W64-LABEL: mubuf_vgpr:
@@ -67,22 +71,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100_W32-LABEL: mubuf_vgpr:
@@ -93,21 +99,23 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v4, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
+; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v1
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100_W64-LABEL: mubuf_vgpr:
@@ -118,21 +126,23 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
; W64-O0-LABEL: mubuf_vgpr:
@@ -249,6 +259,7 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX9_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX9_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -257,8 +268,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX9_W64-NEXT: ; %bb.2:
@@ -275,14 +287,15 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
; GFX9_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9_W64-NEXT: ; implicit-def: $vgpr4
; GFX9_W64-NEXT: ; implicit-def: $vgpr8
+; GFX9_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX9_W64-NEXT: ; %bb.4:
; GFX9_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9_W64-NEXT: global_store_dword v[9:10], v13, off
+; GFX9_W64-NEXT: global_store_dword v[9:10], v1, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: global_store_dword v[11:12], v0, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
@@ -295,15 +308,17 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -320,14 +335,15 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
-; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
+; GFX1010_W32-NEXT: global_store_dword v[9:10], v1, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: global_store_dword v[11:12], v0, off
@@ -341,15 +357,17 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -366,14 +384,15 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
+; GFX1010_W64-NEXT: global_store_dword v[9:10], v1, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: global_store_dword v[11:12], v0, off
@@ -388,15 +407,17 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -413,13 +434,14 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
-; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[9:10], v1, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: global_store_b32 v[11:12], v0, off dlc
@@ -434,15 +456,17 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -459,13 +483,14 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[9:10], v1, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: global_store_b32 v[11:12], v0, off dlc
@@ -707,6 +732,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX9_W64-NEXT: s_mov_b64 s[12:13], exec
; GFX9_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -715,9 +741,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX9_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
; GFX9_W64-NEXT: ; implicit-def: $vgpr8
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX9_W64-NEXT: ; %bb.2:
@@ -739,9 +766,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9_W64-NEXT: buffer_load_format_x v1, v0, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr4
; GFX9_W64-NEXT: ; implicit-def: $vgpr0
+; GFX9_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX9_W64-NEXT: ; %bb.5:
@@ -749,7 +777,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX9_W64-NEXT: .LBB2_6: ; %bb2
; GFX9_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX9_W64-NEXT: global_store_dword v[11:12], v1, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -759,21 +787,23 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -794,10 +824,11 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -805,7 +836,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v1, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -815,21 +846,23 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -850,10 +883,11 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -861,7 +895,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v1, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -871,22 +905,24 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -908,10 +944,11 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -919,7 +956,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v1, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -929,22 +966,24 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -966,10 +1005,11 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -977,7 +1017,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v1, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir
index ef9f4498d7b33..0dcf4ba3fb1b4 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir
+++ b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir
@@ -310,9 +310,7 @@ body: |
; GCN-NEXT: liveins: $sgpr0, $vgpr0_vgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY]], implicit-def dead $scc
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_AND_B32_]], %subreg.sub0, [[S_AND_B32_]], %subreg.sub1
- ; GCN-NEXT: S_CMP_EQ_U32 killed [[REG_SEQUENCE]].sub0, 1, implicit-def $scc
+ ; GCN-NEXT: S_BITCMP1_B32 killed [[COPY]], 0, implicit-def $scc
; GCN-NEXT: S_CBRANCH_SCC0 %bb.2, implicit $scc
; GCN-NEXT: S_BRANCH %bb.1
; GCN-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/or3.ll b/llvm/test/CodeGen/AMDGPU/or3.ll
index bf331ee6c6e3b..71bde22a3fb1b 100644
--- a/llvm/test/CodeGen/AMDGPU/or3.ll
+++ b/llvm/test/CodeGen/AMDGPU/or3.ll
@@ -168,12 +168,6 @@ define <2 x i32> @v_or3_v2i32_b(<2 x i32> inreg %a, <2 x i32> %b, <2 x i32> inre
; GFX9-NEXT: v_or_b32_e32 v1, s19, v1
; GFX9-NEXT: v_or_b32_e32 v0, s18, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_or3_v2i32_b:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DAG: v_or3_b32 v0, s{{[0-9]+}}, v0, s{{[0-9]+}}
-; GFX10-DAG: v_or3_b32 v1, s{{[0-9]+}}, v1, s{{[0-9]+}}
%x = or <2 x i32> %a, %b
%result = or <2 x i32> %x, %c
ret <2 x i32> %result
@@ -195,12 +189,6 @@ define <2 x i32> @v_or3_v2i32_ab(<2 x i32> %a, <2 x i32> %b, <2 x i32> inreg %c)
; GFX9-NEXT: v_or3_b32 v1, v1, v3, s17
; GFX9-NEXT: v_or3_b32 v0, v0, v2, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_or3_v2i32_ab:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DAG: v_or3_b32 v0, v0, v2, s{{[0-9]+}}
-; GFX10-DAG: v_or3_b32 v1, v1, v3, s{{[0-9]+}}
%x = or <2 x i32> %a, %b
%result = or <2 x i32> %x, %c
ret <2 x i32> %result
@@ -248,9 +236,9 @@ define <2 x i32> @v_or3_v2i32_inline_const(<2 x i32> %a, <2 x i32> %b) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x809
+; GFX9-NEXT: s_movk_i32 s5, 0x808
; GFX9-NEXT: v_or3_b32 v1, v1, v3, s4
-; GFX9-NEXT: s_movk_i32 s4, 0x808
-; GFX9-NEXT: v_or3_b32 v0, v0, v2, s4
+; GFX9-NEXT: v_or3_b32 v0, v0, v2, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_or3_v2i32_inline_const:
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
index 6d2f4e76840ae..a60a7550f7fd9 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
@@ -14,7 +14,6 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
%0:vreg_64_align2 = COPY $vgpr0_vgpr1
@@ -38,9 +37,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B32_e32_]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
%0:vreg_64_align2 = COPY $vgpr0_vgpr1
%1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
%2:vreg_64_align2 = REG_SEQUENCE %0.sub0, %subreg.sub0, %1, %subreg.sub1
@@ -62,7 +59,6 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub0, %subreg.sub0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
%0:vreg_64_align2 = COPY $vgpr0_vgpr1
@@ -86,9 +82,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub0, %subreg.sub0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B32_e32_]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
%0:vreg_64_align2 = COPY $vgpr0_vgpr1
%1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
%2:vreg_64_align2 = REG_SEQUENCE %1, %subreg.sub1, %0.sub0, %subreg.sub0
@@ -110,7 +104,6 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub1, %subreg.sub2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
%0:vreg_64 = COPY $vgpr1_vgpr2
@@ -134,7 +127,6 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1, [[COPY]].sub1, %subreg.sub2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
%0:vreg_64 = COPY $vgpr1_vgpr2
@@ -394,7 +386,6 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr1_vgpr2_vgpr3_vgpr4
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]].sub2_sub3, %subreg.sub0_sub1, [[COPY]].sub0, %subreg.sub2, [[V_MOV_B32_e32_1]], %subreg.sub3
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
%0:vreg_128 = COPY $vgpr1_vgpr2_vgpr3_vgpr4
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
index c9645c31aad75..e876631630ce4 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
@@ -21,8 +21,6 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
- ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GCN-NEXT: KILL [[COPY3]], implicit [[COPY2]]
@@ -91,7 +89,6 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GCN-NEXT: $vgpr0 = COPY [[COPY2]]
%0:vgpr_32 = COPY $vgpr0
@@ -155,7 +152,6 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
; GCN-NEXT: $vgpr0 = COPY [[COPY2]]
%0:vreg_64 = COPY $vgpr0_vgpr1
@@ -175,7 +171,6 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]].sub1, %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
; GCN-NEXT: $vgpr0 = COPY [[COPY2]]
%0:vreg_64 = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-regseq-fold.mir b/llvm/test/CodeGen/AMDGPU/peephole-regseq-fold.mir
new file mode 100644
index 0000000000000..ea953ae67e019
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/peephole-regseq-fold.mir
@@ -0,0 +1,309 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=peephole-opt -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=GFX900
+# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx908 -run-pass=peephole-opt -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=GFX900,GFX908
+
+# The optimizer should rewrite uses of REG_SEQUENCE subregisters to use source
+# registers directly when safe.
+
+---
+name: test_regseq_fold_basic
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_basic
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = COPY %2.sub0
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_does_not_look_through_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_does_not_look_through_copy
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; GFX900-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_2]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vreg_64 = COPY %2
+ %4:vgpr_32 = V_MOV_B32_e32 %3.sub0, implicit $exec
+ S_ENDPGM 0, implicit %4
+
+...
+---
+name: test_regseq_fold_does_not_compose_nested_regsequence
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_does_not_compose_nested_regsequence
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
+ ; GFX900-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[V_MOV_B32_e32_2]], %subreg.sub2
+ ; GFX900-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[REG_SEQUENCE1]].sub0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[V_MOV_B32_e32_1]], implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+ %3:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %4:vreg_96 = REG_SEQUENCE %3, %subreg.sub0_sub1, %2, %subreg.sub2
+ %5:vgpr_32 = V_MOV_B32_e32 %4.sub0, implicit $exec
+ %6:vgpr_32 = V_MOV_B32_e32 %3.sub1, implicit $exec
+ S_ENDPGM 0, implicit %5, implicit %6
+
+...
+---
+name: test_regseq_fold_debug_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_debug_use
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: DBG_VALUE $noreg, 0, 0
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = COPY %2.sub0
+ DBG_VALUE %2.sub1, 0, 0
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_undef_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_undef_source
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, undef %1, %subreg.sub1
+ %3:vgpr_32 = COPY %2.sub0
+ %4:vgpr_32 = COPY %2.sub1
+ S_ENDPGM 0, implicit %3, implicit %4
+
+...
+---
+name: test_regseq_fold_clears_kill
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_clears_kill
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = COPY killed %2.sub0
+ %4:vgpr_32 = COPY %0
+ S_ENDPGM 0, implicit %3, implicit %4
+
+...
+---
+name: test_regseq_fold_skip_extra_src_regalloc_req
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; GFX900-LABEL: name: test_regseq_fold_skip_extra_src_regalloc_req
+ ; GFX900: liveins: $vgpr2
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[V_MOV_B32_e32_]], $vgpr2, implicit-def $vcc, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e32_]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = V_ADD_CO_U32_e32 %2.sub0, $vgpr2, implicit-def $vcc, implicit $exec
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_skip_target_illegal_whole_sgpr_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_skip_target_illegal_whole_sgpr_source
+ ; GFX900: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX900-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 42
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
+ ; GFX900-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[REG_SEQUENCE]].sub0, [[S_MOV_B32_1]], 0, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e64_]]
+ %0:sreg_32 = S_MOV_B32 0
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:sreg_32 = S_MOV_B32 42
+ %3:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %4:vgpr_32, dead %5:sreg_64_xexec = V_ADD_CO_U32_e64 %3.sub0, %2, 0, implicit $exec ; folding would violate constant bus restriction
+ S_ENDPGM 0, implicit %4
+
+...
+---
+name: test_regseq_fold_with_copy_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ ; Test that we skip folding when source is a COPY
+ ; GFX900-LABEL: name: test_regseq_fold_with_copy_source
+ ; GFX900: liveins: $vgpr0, $vgpr1
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX900-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX900-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]], implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = V_MOV_B32_e32 %2.sub0, implicit $exec
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_skip_movrels
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_skip_movrels
+ ; GFX900: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX900-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+ ; GFX900-NEXT: [[S_MOVRELS_B32_:%[0-9]+]]:sreg_32 = S_MOVRELS_B32 [[REG_SEQUENCE]].sub0, implicit $m0, implicit [[REG_SEQUENCE]]
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[S_MOVRELS_B32_]]
+ %0:sreg_32 = S_MOV_B32 0
+ %1:sreg_32 = S_MOV_B32 1
+ %2:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:sreg_32 = S_MOVRELS_B32 %2.sub0, implicit $m0, implicit %2
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_skip_inline_asm_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+
+ ; GFX900-LABEL: name: test_regseq_fold_skip_inline_asm_use
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
+ ; GFX900-NEXT: INLINEASM &"; use $0", sideeffect attdialect, reguse:VGPR_32, [[REG_SEQUENCE]].sub0
+ ; GFX900-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ INLINEASM &"; use $0", sideeffect attdialect, reguse:VGPR_32, %2.sub0
+ S_ENDPGM 0
+
+...
+---
+name: test_regseq_fold_skip_phys_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2
+
+ ; GFX900-LABEL: name: test_regseq_fold_skip_phys_source
+ ; GFX900: liveins: $vgpr0, $vgpr2
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
+ ; GFX900-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[REG_SEQUENCE]].sub0, $vgpr2, implicit-def $vcc, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e32_]]
+ %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %1:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, %0, %subreg.sub1
+ %2:vgpr_32 = V_ADD_CO_U32_e32 %1.sub0, $vgpr2, implicit-def $vcc, implicit $exec
+ S_ENDPGM 0, implicit %2
+
+...
+---
+name: test_regseq_fold_with_subreg_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_with_subreg_source
+ ; GFX900: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[COPY]]
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_64 = REG_SEQUENCE %0.sub0, %subreg.sub0, %0.sub1, %subreg.sub1
+ %2:vgpr_32 = COPY %1.sub0
+ S_ENDPGM 0, implicit %2
+
+...
+---
+name: test_regseq_fold_no_subreg_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ ; Test that we skip when use doesn't extract a subregister
+ ; GFX900-LABEL: name: test_regseq_fold_no_subreg_use
+ ; GFX900: liveins: $vgpr0, $vgpr1
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX900-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX900-NEXT: [[V_LSHLREV_B64_e64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64_e64 0, [[REG_SEQUENCE]], implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B64_e64_]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vreg_64 = V_LSHLREV_B64_e64 0, %2, implicit $exec
+ S_ENDPGM 0, implicit %3
+
+...
+---
+name: test_regseq_fold_constrains_source_regclass
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX908-LABEL: name: test_regseq_fold_constrains_source_regclass
+ ; GFX908: [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[V_ACCVGPR_READ_B32_e64_:%[0-9]+]]:vgpr_32 = V_ACCVGPR_READ_B32_e64 [[DEF]], implicit $exec
+ ; GFX908-NEXT: S_ENDPGM 0, implicit [[V_ACCVGPR_READ_B32_e64_]]
+ %0:av_32 = IMPLICIT_DEF
+ %1:areg_64 = REG_SEQUENCE %0, %subreg.sub0
+ %2:vgpr_32 = V_ACCVGPR_READ_B32_e64 %1.sub0, implicit $exec
+ S_ENDPGM 0, implicit %2
+
+...
+---
+name: test_regseq_fold_with_implicit_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX900-LABEL: name: test_regseq_fold_with_implicit_use
+ ; GFX900: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX900-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX900-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %2:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+ %3:vgpr_32 = COPY %2.sub0
+ %4:vgpr_32 = COPY %2.sub1
+ S_ENDPGM 0, implicit %3, implicit %4
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-reg-sequence.mir b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-reg-sequence.mir
new file mode 100644
index 0000000000000..7659c7633b1a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-reg-sequence.mir
@@ -0,0 +1,114 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -mcpu=gfx900 -o - %s | FileCheck --check-prefix=GCN %s
+
+--- |
+ define amdgpu_kernel void @sdwa_reg_sequence_add_shr_i32() {
+ ; GCN-LABEL: sdwa_reg_sequence_add_shr_i32:
+ ; GCN: ; %bb.0:
+ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; GCN-NEXT: flat_load_dword v2, v[2:3] glc
+ ; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+ ; GCN-NEXT: buffer_wbinvl1_vol
+ ; GCN-NEXT: v_mov_b32_e32 v3, 0x7b
+ ; GCN-NEXT: v_add_co_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+ ; GCN-NEXT: flat_store_dword v[0:1], v2
+ ; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+ ; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret void
+ }
+
+ define amdgpu_kernel void @sdwa_reg_sequence_and_add_co() {
+ ; GCN-LABEL: sdwa_reg_sequence_and_add_co:
+ ; GCN: ; %bb.0:
+ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; GCN-NEXT: v_add_u32_e32 v1, 10, v0
+ ; GCN-NEXT: v_add_u32_e32 v0, 20, v0
+ ; GCN-NEXT: v_add_co_u32_sdwa v0, vcc, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+ ; GCN-NEXT: v_addc_co_u32_e64 v1, s[0:1], 0, 0, vcc
+ ; GCN-NEXT: global_store_dwordx2 v[0:1], v[0:1], off
+ ; GCN-NEXT: s_endpgm
+ ret void
+ }
+
+ define amdgpu_kernel void @sdwa_reg_sequence_composed_subregs() {
+ ; GCN-LABEL: sdwa_reg_sequence_composed_subregs:
+ ; GCN: ; %bb.0:
+ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; GCN-NEXT: v_add_u32_e32 v0, 20, v2
+ ; GCN-NEXT: v_mov_b32_e32 v1, 0
+ ; GCN-NEXT: v_add_co_u32_sdwa v0, vcc, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+ ; GCN-NEXT: v_addc_co_u32_e64 v1, s[0:1], 0, 0, vcc
+ ; GCN-NEXT: global_store_dwordx2 v[0:1], v[0:1], off
+ ; GCN-NEXT: s_endpgm
+ ret void
+ }
+...
+
+---
+name: sdwa_reg_sequence_add_shr_i32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr30_sgpr31
+
+ %0:vreg_64 = COPY $vgpr0_vgpr1
+ %1:vreg_64 = COPY $vgpr2_vgpr3
+ %2:sreg_64 = COPY $sgpr30_sgpr31
+ %3:vgpr_32 = FLAT_LOAD_DWORD %1, 0, 0, implicit $exec, implicit $flat_scr
+ %4:sreg_32_xm0 = S_MOV_B32 123
+ %5:vgpr_32 = V_LSHRREV_B32_e64 16, %3, implicit $exec
+ %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %7:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %6, %subreg.sub1
+ %8:vgpr_32 = V_ADD_CO_U32_e32 %4, killed %7.sub0, implicit-def $vcc, implicit $exec
+ FLAT_STORE_DWORD %0, %8, 0, 0, implicit $exec, implicit $flat_scr
+ $sgpr30_sgpr31 = COPY %2
+ S_SETPC_B64_return $sgpr30_sgpr31
+...
+
+---
+name: sdwa_reg_sequence_and_add_co
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_ADD_U32_e64 %0, 10, 0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e64 %0, 20, 0, implicit $exec
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %4:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %3, %subreg.sub1
+ %5:sreg_32 = S_MOV_B32 255
+ %6:vgpr_32 = V_AND_B32_e64 killed %2, killed %5, implicit $exec
+ %7:vreg_64 = REG_SEQUENCE %6, %subreg.sub0, %3, %subreg.sub1
+ %8:vgpr_32, %9:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %7.sub0, 0, implicit $exec
+ %10:vgpr_32, dead %11:sreg_64_xexec = V_ADDC_U32_e64 0, 0, killed %9, 0, implicit $exec
+ %12:vreg_64 = REG_SEQUENCE %8, %subreg.sub0, %10, %subreg.sub1
+ %13:sreg_64 = IMPLICIT_DEF
+ %14:vreg_64 = COPY %13
+ GLOBAL_STORE_DWORDX2 killed %14, killed %12, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: sdwa_reg_sequence_composed_subregs
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1_vgpr2
+
+ %0:vreg_64 = COPY $vgpr1_vgpr2
+ %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, 10, 0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e64 %0.sub1, 20, 0, implicit $exec
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %4:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %3, %subreg.sub1
+ %5:vreg_64 = REG_SEQUENCE %0.sub0, %subreg.sub0, %4.sub1, %subreg.sub1
+ %6:sreg_32 = S_MOV_B32 255
+ %7:vgpr_32 = V_AND_B32_e64 killed %2, killed %6, implicit $exec
+ %8:vreg_64 = REG_SEQUENCE %7, %subreg.sub0, %3, %subreg.sub1
+ %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %5.sub1, %8.sub0, 0, implicit $exec
+ %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 0, 0, killed %10, 0, implicit $exec
+ %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+ %15:vreg_64 = COPY %13
+ GLOBAL_STORE_DWORDX2 killed %15, killed %13, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sibling-call.ll b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
index 862ed39078fea..169dee095d2de 100644
--- a/llvm/test/CodeGen/AMDGPU/sibling-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
@@ -648,8 +648,8 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; FIJI-NEXT: v_mov_b32_e32 v0, v2
; FIJI-NEXT: v_mov_b32_e32 v1, v3
; FIJI-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; FIJI-NEXT: v_mov_b32_e32 v4, v0
-; FIJI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; FIJI-NEXT: v_mov_b32_e32 v1, v0
+; FIJI-NEXT: ; implicit-def: $vgpr0
; FIJI-NEXT: ; implicit-def: $vgpr31
; FIJI-NEXT: ; implicit-def: $vgpr2
; FIJI-NEXT: ; implicit-def: $vgpr3
@@ -658,7 +658,7 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; FIJI-NEXT: ; %bb.2:
; FIJI-NEXT: s_mov_b64 exec, s[54:55]
; FIJI-NEXT: v_readlane_b32 s30, v40, 16
-; FIJI-NEXT: v_mov_b32_e32 v0, v4
+; FIJI-NEXT: v_mov_b32_e32 v0, v1
; FIJI-NEXT: v_readlane_b32 s31, v40, 17
; FIJI-NEXT: v_readlane_b32 s65, v40, 15
; FIJI-NEXT: v_readlane_b32 s64, v40, 14
@@ -739,8 +739,8 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; HAWAII-NEXT: v_mov_b32_e32 v0, v2
; HAWAII-NEXT: v_mov_b32_e32 v1, v3
; HAWAII-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; HAWAII-NEXT: v_mov_b32_e32 v4, v0
-; HAWAII-NEXT: ; implicit-def: $vgpr0_vgpr1
+; HAWAII-NEXT: v_mov_b32_e32 v1, v0
+; HAWAII-NEXT: ; implicit-def: $vgpr0
; HAWAII-NEXT: ; implicit-def: $vgpr31
; HAWAII-NEXT: ; implicit-def: $vgpr2
; HAWAII-NEXT: ; implicit-def: $vgpr3
@@ -749,7 +749,7 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; HAWAII-NEXT: ; %bb.2:
; HAWAII-NEXT: s_mov_b64 exec, s[54:55]
; HAWAII-NEXT: v_readlane_b32 s30, v40, 16
-; HAWAII-NEXT: v_mov_b32_e32 v0, v4
+; HAWAII-NEXT: v_mov_b32_e32 v0, v1
; HAWAII-NEXT: v_readlane_b32 s31, v40, 17
; HAWAII-NEXT: v_readlane_b32 s65, v40, 15
; HAWAII-NEXT: v_readlane_b32 s64, v40, 14
@@ -830,8 +830,8 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr31
; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: ; implicit-def: $vgpr3
@@ -840,7 +840,7 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
; GFX9-NEXT: ; %bb.2:
; GFX9-NEXT: s_mov_b64 exec, s[54:55]
; GFX9-NEXT: v_readlane_b32 s30, v40, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
; GFX9-NEXT: v_readlane_b32 s31, v40, 17
; GFX9-NEXT: v_readlane_b32 s65, v40, 15
; GFX9-NEXT: v_readlane_b32 s64, v40, 14
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 8352e3948611b..9209ba2fab3fd 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -242,7 +242,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.2.if:
; SI-NEXT: successors: %bb.3(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[PHI2]], %subreg.sub0, killed [[PHI3]], %subreg.sub1
+ ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[PHI2]], %subreg.sub0, [[PHI3]], %subreg.sub1
; SI-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: [[COPY6:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
; SI-NEXT: {{ $}}
@@ -250,19 +250,21 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI4:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %41, %bb.4
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %58:vgpr_32, %bb.4, [[PHI1]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %56:vgpr_32, %bb.4, [[PHI2]], %bb.2
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %58:vgpr_32, %bb.4, [[PHI3]], %bb.2
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:vreg_64 = PHI undef %60:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[PHI8:%[0-9]+]]:vgpr_32 = PHI undef %62:vgpr_32, %bb.4, [[PHI1]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI5]], implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI6]], implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI5]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI7]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY6]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI6]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI8]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
@@ -279,31 +281,33 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.6.else:
; SI-NEXT: successors: %bb.7(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY1]], %subreg.sub0, killed [[COPY]], %subreg.sub1
+ ; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
; SI-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI7:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %33, %bb.8
- ; SI-NEXT: [[PHI8:%[0-9]+]]:vreg_64 = PHI undef %60:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %62:vgpr_32, %bb.8, [[COPY4]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI9:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %33, %bb.8
+ ; SI-NEXT: [[PHI10:%[0-9]+]]:vgpr_32 = PHI undef %64:vgpr_32, %bb.8, [[COPY1]], %bb.6
+ ; SI-NEXT: [[PHI11:%[0-9]+]]:vgpr_32 = PHI undef %66:vgpr_32, %bb.8, [[COPY]], %bb.6
+ ; SI-NEXT: [[PHI12:%[0-9]+]]:vreg_64 = PHI undef %68:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[PHI13:%[0-9]+]]:vgpr_32 = PHI undef %70:vgpr_32, %bb.8, [[COPY4]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI10]], implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI11]], implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI8]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI12]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY9]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI9]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI13]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI9]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -314,9 +318,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI10:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI14:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI10]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI14]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
%cc = icmp sgt i32 %z, 5
@@ -364,7 +368,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.2.if:
; SI-NEXT: successors: %bb.3(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[PHI1]], %subreg.sub0, killed [[PHI2]], %subreg.sub1
+ ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[PHI1]], %subreg.sub0, [[PHI2]], %subreg.sub1
; SI-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: [[COPY6:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
; SI-NEXT: {{ $}}
@@ -372,11 +376,13 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI3:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %42, %bb.4
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:vgpr_32 = PHI undef %55:vgpr_32, %bb.4, [[PHI1]], %bb.2
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %57:vgpr_32, %bb.4, [[PHI2]], %bb.2
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %59:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI4]], implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI5]], implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI4]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI6]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
@@ -400,19 +406,21 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.6.else:
; SI-NEXT: successors: %bb.7(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY1]], %subreg.sub0, killed [[COPY]], %subreg.sub1
+ ; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
; SI-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %34, %bb.8
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %34, %bb.8
+ ; SI-NEXT: [[PHI8:%[0-9]+]]:vgpr_32 = PHI undef %61:vgpr_32, %bb.8, [[COPY1]], %bb.6
+ ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %63:vgpr_32, %bb.8, [[COPY]], %bb.6
+ ; SI-NEXT: [[PHI10:%[0-9]+]]:vreg_64 = PHI undef %65:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI8]], implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[PHI9]], implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI6]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI10]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
@@ -423,7 +431,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -434,9 +442,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI11:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI7]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
+ ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI11]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
; SI-NEXT: $vgpr0 = COPY killed [[V_ADD_F32_e64_]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index 9e0db1dfa26c5..b165220f86cd0 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -188,7 +188,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
-; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_cbranch_execnz .LBB3_2
; SI-NEXT: ; %bb.3:
@@ -211,7 +211,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
-; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_cbranch_execnz .LBB3_6
; SI-NEXT: ; %bb.7:
@@ -267,7 +267,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
-; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_cbranch_execnz .LBB4_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -288,7 +288,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
-; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_cbranch_execnz .LBB4_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
index e67f992380faa..4337f7f46d798 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
+++ b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
@@ -24,10 +24,10 @@ define amdgpu_kernel void @foo(i1 %cmp1) {
; GFX906-NEXT: s_mov_b32 s15, 0xe00000
; GFX906-NEXT: s_add_u32 s12, s12, s11
; GFX906-NEXT: s_addc_u32 s13, s13, 0
-; GFX906-NEXT: buffer_load_dword v3, off, s[12:15], 0
-; GFX906-NEXT: buffer_load_dword v4, off, s[12:15], 0 offset:4
-; GFX906-NEXT: buffer_load_dword v5, off, s[12:15], 0 offset:8
-; GFX906-NEXT: buffer_load_dword v6, off, s[12:15], 0 offset:12
+; GFX906-NEXT: buffer_load_dword v5, off, s[12:15], 0
+; GFX906-NEXT: buffer_load_dword v6, off, s[12:15], 0 offset:4
+; GFX906-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8
+; GFX906-NEXT: buffer_load_dword v4, off, s[12:15], 0 offset:12
; GFX906-NEXT: s_load_dword s2, s[4:5], 0x24
; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x1c
; GFX906-NEXT: s_mov_b32 s4, 0
@@ -44,18 +44,19 @@ define amdgpu_kernel void @foo(i1 %cmp1) {
; GFX906-NEXT: ds_write_b64 v2, v[0:1]
; GFX906-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX906-NEXT: s_waitcnt vmcnt(3)
-; GFX906-NEXT: v_readfirstlane_b32 s0, v3
+; GFX906-NEXT: v_readfirstlane_b32 s0, v5
; GFX906-NEXT: s_waitcnt vmcnt(2)
-; GFX906-NEXT: v_readfirstlane_b32 s1, v4
-; GFX906-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[3:4]
+; GFX906-NEXT: v_readfirstlane_b32 s1, v6
+; GFX906-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[5:6]
; GFX906-NEXT: s_waitcnt vmcnt(1)
-; GFX906-NEXT: v_readfirstlane_b32 s0, v5
+; GFX906-NEXT: v_readfirstlane_b32 s0, v3
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: v_readfirstlane_b32 s1, v6
-; GFX906-NEXT: v_cmp_eq_u64_e64 s[0:1], s[0:1], v[5:6]
+; GFX906-NEXT: v_readfirstlane_b32 s1, v4
+; GFX906-NEXT: v_cmp_eq_u64_e64 s[0:1], s[0:1], v[3:4]
; GFX906-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX906-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX906-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5_vgpr6
+; GFX906-NEXT: ; implicit-def: $vgpr5
+; GFX906-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX906-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX906-NEXT: s_cbranch_execnz .LBB0_1
; GFX906-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/Thumb2/mve-clmul.ll b/llvm/test/CodeGen/Thumb2/mve-clmul.ll
index aacba93e117ea..0c9f1bfbd14cb 100644
--- a/llvm/test/CodeGen/Thumb2/mve-clmul.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-clmul.ll
@@ -6246,30 +6246,29 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: sub sp, #496
; CHECK-NEXT: vmov q2, q0
; CHECK-NEXT: movs r3, #0
-; CHECK-NEXT: vmov.f32 s14, s9
+; CHECK-NEXT: vmov.f32 s2, s9
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r0, s8
; CHECK-NEXT: vmov q6, q1
-; CHECK-NEXT: vmov.f32 s16, s24
+; CHECK-NEXT: vmov.f32 s12, s24
; CHECK-NEXT: mov.w lr, #0
-; CHECK-NEXT: vmov.f32 s18, s25
-; CHECK-NEXT: vmov.f32 s12, s8
-; CHECK-NEXT: vmov r12, s14
+; CHECK-NEXT: vmov.f32 s14, s25
+; CHECK-NEXT: vmov r12, s2
; CHECK-NEXT: mov r4, r0
; CHECK-NEXT: lsll r4, r5, #1
; CHECK-NEXT: mov r2, r12
; CHECK-NEXT: lsll r2, r3, #1
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_32
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: vstrw.32 q1, [sp, #480] @ 16-byte Spill
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vmov.i64 q1, #0xffffffff
-; CHECK-NEXT: vand q3, q3, q1
+; CHECK-NEXT: vand q0, q0, q1
; CHECK-NEXT: vmov.i32 q1, #0x0
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: lsll r4, r5, #2
@@ -6283,10 +6282,10 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: adr.w r2, .LCPI22_33
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
-; CHECK-NEXT: vpsel q3, q1, q3
+; CHECK-NEXT: vpsel q0, q1, q0
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: veor q0, q3, q0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: veor q4, q0, q4
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: vstrw.32 q1, [sp, #464] @ 16-byte Spill
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: cmp r2, #0
@@ -6300,18 +6299,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #2
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_34
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #448] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #3
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6324,18 +6323,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #3
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_35
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #432] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #4
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6348,18 +6347,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #4
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_36
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #416] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #5
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6372,18 +6371,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #5
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_37
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #400] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #6
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6396,18 +6395,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #6
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_38
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #384] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #7
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6420,18 +6419,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #7
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_39
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #368] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #8
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6444,18 +6443,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #8
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_40
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #352] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #9
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6468,18 +6467,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #9
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_41
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #336] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #10
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6492,18 +6491,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #10
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_42
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #320] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #11
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6516,18 +6515,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #11
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_43
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #304] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #12
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6540,18 +6539,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #12
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_44
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #288] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #13
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6564,18 +6563,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #13
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_45
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #272] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #14
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6588,18 +6587,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #14
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_14
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #256] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #15
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6612,18 +6611,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #15
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_15
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #240] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #16
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6636,18 +6635,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #16
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_16
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #224] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #17
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6660,18 +6659,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #17
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_17
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #208] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #18
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6684,18 +6683,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #18
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_18
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #192] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #19
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6708,18 +6707,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #19
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_19
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #176] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #20
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6732,18 +6731,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #20
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_20
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #160] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #21
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6756,18 +6755,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #21
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_21
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #144] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #22
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6780,18 +6779,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #22
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_22
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #128] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #23
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6804,18 +6803,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #23
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_23
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #24
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6828,18 +6827,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #24
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_24
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #96] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #25
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6852,18 +6851,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #25
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_25
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #26
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6876,18 +6875,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #26
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_26
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #27
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6900,18 +6899,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #27
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_27
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #28
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6924,18 +6923,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #28
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_28
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #29
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6948,18 +6947,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #29
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q0, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_29
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: vand q5, q4, q1
+; CHECK-NEXT: vand q5, q3, q1
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r2, s20
; CHECK-NEXT: vstrw.32 q1, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT: veor q0, q3, q0
+; CHECK-NEXT: veor q4, q0, q4
; CHECK-NEXT: lsll r4, r5, #30
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
@@ -6972,17 +6971,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r2, r3, #30
-; CHECK-NEXT: vpsel q3, q3, q0
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT: vpsel q1, q0, q4
+; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_30
; CHECK-NEXT: vldrw.u32 q7, [r2]
-; CHECK-NEXT: vmov q0[3], q0[1], r5, r3
-; CHECK-NEXT: veor q1, q3, q0
+; CHECK-NEXT: vmov q4[3], q4[1], r5, r3
; CHECK-NEXT: movs r3, #0
-; CHECK-NEXT: vand q0, q4, q7
; CHECK-NEXT: movs r5, #0
+; CHECK-NEXT: vand q0, q3, q7
+; CHECK-NEXT: veor q4, q1, q4
; CHECK-NEXT: vmov r2, s0
; CHECK-NEXT: lsll r0, r5, #31
+; CHECK-NEXT: vmov r4, s10
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: csetm r2, eq
; CHECK-NEXT: bfi r3, r2, #0, #8
@@ -6994,43 +6994,42 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vmsr p0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: lsll r12, r3, #31
-; CHECK-NEXT: vpsel q0, q3, q1
+; CHECK-NEXT: vpsel q0, q1, q4
; CHECK-NEXT: vmov q1[2], q1[0], r0, r12
; CHECK-NEXT: adr.w r0, .LCPI22_31
; CHECK-NEXT: vldrw.u32 q5, [r0]
; CHECK-NEXT: vmov q1[3], q1[1], r5, r3
; CHECK-NEXT: veor q1, q0, q1
-; CHECK-NEXT: vmov.i64 q3, #0xffffffff
-; CHECK-NEXT: vand q4, q4, q5
-; CHECK-NEXT: vmov r0, s16
-; CHECK-NEXT: vmov.f32 s16, s26
+; CHECK-NEXT: vmov.i64 q4, #0xffffffff
+; CHECK-NEXT: vand q3, q3, q5
+; CHECK-NEXT: vmov r12, s11
+; CHECK-NEXT: vmov r0, s12
+; CHECK-NEXT: vmov.f32 s12, s26
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #0, #8
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: vmov.f32 s18, s27
-; CHECK-NEXT: vand q4, q4, q3
+; CHECK-NEXT: vmov r0, s14
+; CHECK-NEXT: vmov.f32 s14, s27
+; CHECK-NEXT: vand q3, q3, q4
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #8, #8
+; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: vmsr p0, r2
-; CHECK-NEXT: vldrw.u32 q6, [sp, #480] @ 16-byte Reload
+; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: vpsel q0, q0, q1
-; CHECK-NEXT: vmov.f32 s4, s10
-; CHECK-NEXT: vmov.f32 s6, s11
+; CHECK-NEXT: vldrw.u32 q6, [sp, #480] @ 16-byte Reload
; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT: vand q0, q1, q3
-; CHECK-NEXT: vand q6, q4, q6
+; CHECK-NEXT: vmov.f32 s0, s10
+; CHECK-NEXT: vmov.f32 s2, s11
+; CHECK-NEXT: vand q6, q3, q6
+; CHECK-NEXT: vand q0, q0, q4
; CHECK-NEXT: vmov r3, s3
; CHECK-NEXT: vmov r1, s1
-; CHECK-NEXT: vmov r4, s4
-; CHECK-NEXT: vmov r12, s6
; CHECK-NEXT: mov r7, r3
; CHECK-NEXT: mov r5, r1
-; CHECK-NEXT: mov r2, r4
-; CHECK-NEXT: mov r0, r12
-; CHECK-NEXT: lsll r2, r5, #2
; CHECK-NEXT: lsll r0, r7, #2
+; CHECK-NEXT: lsll r2, r5, #2
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: vmov q1[3], q1[1], r5, r7
@@ -7058,7 +7057,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vpsel q0, q6, q0
; CHECK-NEXT: vldrw.u32 q6, [sp, #464] @ 16-byte Reload
; CHECK-NEXT: veor q2, q0, q2
-; CHECK-NEXT: vand q6, q4, q6
+; CHECK-NEXT: vand q6, q3, q6
; CHECK-NEXT: vmov r0, s24
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
@@ -7072,7 +7071,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: vpsel q0, q0, q2
; CHECK-NEXT: vldrw.u32 q2, [sp, #448] @ 16-byte Reload
; CHECK-NEXT: veor q1, q0, q1
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vmov r0, s8
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
@@ -7087,7 +7086,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #3
; CHECK-NEXT: lsll r2, r7, #3
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7109,7 +7108,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #4
; CHECK-NEXT: lsll r2, r7, #4
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7131,7 +7130,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #5
; CHECK-NEXT: lsll r2, r7, #5
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7153,7 +7152,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #6
; CHECK-NEXT: lsll r2, r7, #6
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7175,7 +7174,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #7
; CHECK-NEXT: lsll r2, r7, #7
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7197,7 +7196,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #8
; CHECK-NEXT: lsll r2, r7, #8
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7219,7 +7218,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #9
; CHECK-NEXT: lsll r2, r7, #9
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7241,7 +7240,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #10
; CHECK-NEXT: lsll r2, r7, #10
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7263,7 +7262,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #11
; CHECK-NEXT: lsll r2, r7, #11
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7285,7 +7284,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #12
; CHECK-NEXT: lsll r2, r7, #12
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7307,7 +7306,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #13
; CHECK-NEXT: lsll r2, r7, #13
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7329,7 +7328,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #14
; CHECK-NEXT: lsll r2, r7, #14
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7351,7 +7350,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #15
; CHECK-NEXT: lsll r2, r7, #15
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7373,7 +7372,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #16
; CHECK-NEXT: lsll r2, r7, #16
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7395,7 +7394,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #17
; CHECK-NEXT: lsll r2, r7, #17
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7417,7 +7416,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #18
; CHECK-NEXT: lsll r2, r7, #18
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7439,7 +7438,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #19
; CHECK-NEXT: lsll r2, r7, #19
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7562,7 +7561,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #20
; CHECK-NEXT: lsll r2, r7, #20
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7584,7 +7583,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #21
; CHECK-NEXT: lsll r2, r7, #21
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7606,7 +7605,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #22
; CHECK-NEXT: lsll r2, r7, #22
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7628,7 +7627,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #23
; CHECK-NEXT: lsll r2, r7, #23
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7650,7 +7649,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #24
; CHECK-NEXT: lsll r2, r7, #24
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7672,7 +7671,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #25
; CHECK-NEXT: lsll r2, r7, #25
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7694,7 +7693,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #26
; CHECK-NEXT: lsll r2, r7, #26
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7716,7 +7715,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #27
; CHECK-NEXT: lsll r2, r7, #27
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7738,7 +7737,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #28
; CHECK-NEXT: lsll r2, r7, #28
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7760,7 +7759,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: mov r2, r4
; CHECK-NEXT: lsll r0, r5, #29
; CHECK-NEXT: lsll r2, r7, #29
-; CHECK-NEXT: vand q2, q4, q2
+; CHECK-NEXT: vand q2, q3, q2
; CHECK-NEXT: vpsel q0, q0, q1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r0
; CHECK-NEXT: vmov r0, s8
@@ -7773,7 +7772,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #0, #8
; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vand q2, q4, q7
+; CHECK-NEXT: vand q2, q3, q7
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #8, #8
@@ -7794,7 +7793,7 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #0, #8
; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vand q2, q4, q5
+; CHECK-NEXT: vand q2, q3, q5
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, eq
; CHECK-NEXT: bfi r2, r0, #8, #8
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
index 577c78309cd48..e4a75cfbead0d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
@@ -4,8 +4,8 @@
define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
; CHECK-LABEL: loads_i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT: .save {r4, r5, r6, r7, lr}
+; CHECK-NEXT: push {r4, r5, r6, r7, lr}
; CHECK-NEXT: vldrw.u32 q3, [r1]
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: vmov.i64 q2, #0xffffffff
@@ -22,33 +22,31 @@ define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
; CHECK-NEXT: adds.w r12, r0, r1
; CHECK-NEXT: vmov r0, s7
; CHECK-NEXT: adc.w r1, r6, r5
-; CHECK-NEXT: adds.w r8, r0, r4
-; CHECK-NEXT: vmov r4, r6, d4
-; CHECK-NEXT: asr.w r5, r0, #31
+; CHECK-NEXT: asrs r5, r0, #31
+; CHECK-NEXT: adds r0, r0, r4
; CHECK-NEXT: adcs r5, r3
; CHECK-NEXT: vmov r3, s6
+; CHECK-NEXT: vmov r4, r6, d4
; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s2, s7
+; CHECK-NEXT: vmov r2, s6
; CHECK-NEXT: vmov.f32 s6, s5
-; CHECK-NEXT: vmov r2, s8
; CHECK-NEXT: adds r4, r4, r3
; CHECK-NEXT: asr.w r7, r3, #31
; CHECK-NEXT: adc.w r3, r7, r6
+; CHECK-NEXT: asr.w r7, lr, #31
; CHECK-NEXT: asrl r4, r3, r2
-; CHECK-NEXT: asr.w r2, lr, #31
; CHECK-NEXT: vmov r3, r6, d0
-; CHECK-NEXT: adds.w r0, lr, r3
-; CHECK-NEXT: adc.w r3, r2, r6
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: asrl r0, r3, r2
-; CHECK-NEXT: vmov r2, s2
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
-; CHECK-NEXT: vmov r0, s6
-; CHECK-NEXT: asrl r8, r5, r2
-; CHECK-NEXT: asrl r12, r1, r0
-; CHECK-NEXT: vmov q0[3], q0[1], r12, r8
-; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT: adds.w r2, lr, r3
+; CHECK-NEXT: adc.w r3, r7, r6
+; CHECK-NEXT: vmov r7, s4
+; CHECK-NEXT: asrl r2, r3, r7
+; CHECK-NEXT: vmov r3, s7
+; CHECK-NEXT: vmov q0[2], q0[0], r2, r4
+; CHECK-NEXT: vmov r2, s6
+; CHECK-NEXT: asrl r0, r5, r3
+; CHECK-NEXT: asrl r12, r1, r2
+; CHECK-NEXT: vmov q0[3], q0[1], r12, r0
+; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
entry:
%a = load <4 x i32>, ptr %A, align 4
%b = load <4 x i32>, ptr %B, align 4
@@ -146,19 +144,17 @@ define arm_aapcs_vfpcc void @load_store_i32(ptr %A, ptr %B, ptr %C, ptr %D) {
; CHECK-NEXT: vldrw.u32 q1, [r2]
; CHECK-NEXT: vmov r9, r8, d0
; CHECK-NEXT: vmov r0, s11
-; CHECK-NEXT: vmov.f32 s0, s6
-; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: asrs r2, r0, #31
; CHECK-NEXT: adds r0, r0, r1
; CHECK-NEXT: adc.w r11, r2, r12
; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: vmov.f32 s10, s7
; CHECK-NEXT: asrs r6, r2, #31
; CHECK-NEXT: adds.w r2, r2, lr
; CHECK-NEXT: adc.w r7, r6, r4
; CHECK-NEXT: vmov r4, r5, d4
-; CHECK-NEXT: vmov r6, s0
+; CHECK-NEXT: vmov r6, s6
; CHECK-NEXT: asrl r2, r7, r6
+; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: adds.w r6, r4, r9
; CHECK-NEXT: asr.w r7, r4, #31
; CHECK-NEXT: vmov r4, s4
@@ -167,7 +163,7 @@ define arm_aapcs_vfpcc void @load_store_i32(ptr %A, ptr %B, ptr %C, ptr %D) {
; CHECK-NEXT: vmov r4, r1, d1
; CHECK-NEXT: vmov q0[2], q0[0], r6, r2
; CHECK-NEXT: asrs r2, r5, #31
-; CHECK-NEXT: vmov r7, s10
+; CHECK-NEXT: vmov r7, s7
; CHECK-NEXT: asrl r0, r11, r7
; CHECK-NEXT: adds r6, r5, r4
; CHECK-NEXT: adcs r1, r2
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
index fe5d7f29f01ff..d9ed7fe548c5e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
@@ -64,16 +64,12 @@ entry:
define arm_aapcs_vfpcc <4 x i32> @ext_add_trunc_i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: ext_add_trunc_i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s6, s7
-; CHECK-NEXT: vmov r0, s8
-; CHECK-NEXT: vmov.f32 s8, s2
-; CHECK-NEXT: vmov.f32 s2, s3
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r2, s2
+; CHECK-NEXT: vmov r1, s2
+; CHECK-NEXT: vmov r0, s6
+; CHECK-NEXT: vmov r2, s3
; CHECK-NEXT: vmov.f32 s2, s5
; CHECK-NEXT: add.w r12, r1, r0
-; CHECK-NEXT: vmov r1, s6
+; CHECK-NEXT: vmov r1, s7
; CHECK-NEXT: vmov r0, s0
; CHECK-NEXT: add r1, r2
; CHECK-NEXT: vmov r2, s2
@@ -184,19 +180,17 @@ define arm_aapcs_vfpcc <4 x i32> @ext_add_ashr_trunc_i32(<4 x i32> %a, <4 x i32>
; CHECK-NEXT: push {r4, r5, r6, r7, lr}
; CHECK-NEXT: vmov.f32 s12, s6
; CHECK-NEXT: vmov.i64 q2, #0xffffffff
-; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: vmov.f32 s14, s7
-; CHECK-NEXT: vand q1, q1, q2
-; CHECK-NEXT: vmov r3, r7, d2
; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov.f32 s4, s2
+; CHECK-NEXT: vmov r2, s2
; CHECK-NEXT: vmov r0, r1, d6
-; CHECK-NEXT: vmov.f32 s2, s3
+; CHECK-NEXT: vmov.f32 s6, s5
+; CHECK-NEXT: vand q1, q1, q2
; CHECK-NEXT: vmov lr, r12, d7
-; CHECK-NEXT: vmov r2, s4
+; CHECK-NEXT: vmov r3, r7, d2
; CHECK-NEXT: asrs r5, r2, #31
; CHECK-NEXT: adds r2, r2, r0
-; CHECK-NEXT: vmov r0, s2
+; CHECK-NEXT: vmov r0, s3
; CHECK-NEXT: adcs r1, r5
; CHECK-NEXT: vmov r5, s0
; CHECK-NEXT: asrl r2, r1, #1
@@ -302,26 +296,24 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b)
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr}
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT: vmov.f32 s12, s2
-; CHECK-NEXT: vmov.f32 s2, s3
-; CHECK-NEXT: vmov.f32 s10, s7
+; CHECK-NEXT: vmov r10, s3
+; CHECK-NEXT: vmov.i64 q3, #0xffffffff
+; CHECK-NEXT: vmov r2, s7
; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s6, s5
-; CHECK-NEXT: vmov r10, s2
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: vmov.f32 s2, s1
+; CHECK-NEXT: vmov.f32 s10, s7
+; CHECK-NEXT: vand q2, q2, q3
; CHECK-NEXT: adds.w r6, r10, r2
; CHECK-NEXT: asr.w r0, r10, #31
; CHECK-NEXT: adc r3, r0, #0
; CHECK-NEXT: eor.w r1, r10, r2
; CHECK-NEXT: asrl r6, r3, r2
; CHECK-NEXT: subs r0, r6, r2
-; CHECK-NEXT: vmov r6, s12
+; CHECK-NEXT: vmov r6, s2
; CHECK-NEXT: sbc lr, r3, #0
-; CHECK-NEXT: vmov r3, s8
+; CHECK-NEXT: vmov r3, s6
; CHECK-NEXT: umull r0, r8, r0, r2
-; CHECK-NEXT: vmov.i64 q3, #0xffffffff
-; CHECK-NEXT: vand q2, q2, q3
+; CHECK-NEXT: vmov.f32 s2, s1
+; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: adds r4, r6, r3
; CHECK-NEXT: asr.w r7, r6, #31
; CHECK-NEXT: adc r5, r7, #0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
index dc0a33f483b36..21f5439cecf94 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
@@ -533,23 +533,23 @@ define void @vabd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y,
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vldrw.u32 q6, [r0], #16
; CHECK-NEXT: vldrw.u32 q5, [r1], #16
+; CHECK-NEXT: vmov r12, s22
+; CHECK-NEXT: vmov r3, s26
+; CHECK-NEXT: vmov r4, s24
; CHECK-NEXT: vmov.f32 s12, s22
; CHECK-NEXT: vmov.f32 s16, s26
-; CHECK-NEXT: vmov r4, s24
-; CHECK-NEXT: vmov.f32 s14, s23
-; CHECK-NEXT: vmov.f32 s18, s27
; CHECK-NEXT: vmov.f32 s22, s21
; CHECK-NEXT: vmov.f32 s26, s25
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r3, s16
+; CHECK-NEXT: vmov.f32 s14, s23
+; CHECK-NEXT: vmov.f32 s18, s27
+; CHECK-NEXT: vand q3, q3, q0
+; CHECK-NEXT: vand q4, q4, q0
; CHECK-NEXT: sub.w r12, r3, r12
; CHECK-NEXT: vmov r3, s20
; CHECK-NEXT: subs r3, r4, r3
-; CHECK-NEXT: vmov r4, s18
+; CHECK-NEXT: vmov r4, s27
; CHECK-NEXT: vmov q2[2], q2[0], r3, r12
-; CHECK-NEXT: vmov r3, s14
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
+; CHECK-NEXT: vmov r3, s23
; CHECK-NEXT: sub.w r12, r4, r3
; CHECK-NEXT: vmov r4, s22
; CHECK-NEXT: vmov r3, s26
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll b/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll
index cebc0d9c0e172..e68f9877a3e94 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll
@@ -242,12 +242,10 @@ define arm_aapcs_vfpcc <4 x i64> @sext32_0213_ext0(<8 x i32> %src1, i32 %src2) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r7, lr}
; CHECK-NEXT: push {r4, r5, r7, lr}
-; CHECK-NEXT: vmov.f32 s4, s1
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r1, s6
-; CHECK-NEXT: umull r2, r5, r3, r0
+; CHECK-NEXT: vmov r1, s3
+; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: umull lr, r12, r1, r0
+; CHECK-NEXT: umull r2, r5, r3, r0
; CHECK-NEXT: vmov q1[2], q1[0], r2, lr
; CHECK-NEXT: asrs r2, r0, #31
; CHECK-NEXT: mla r4, r1, r2, r12
@@ -285,13 +283,11 @@ define arm_aapcs_vfpcc <4 x i64> @sext32_ext0_0213(<8 x i32> %src1, i32 %src2) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r7, lr}
; CHECK-NEXT: push {r4, r5, r7, lr}
-; CHECK-NEXT: vmov.f32 s4, s1
+; CHECK-NEXT: vmov r1, s3
; CHECK-NEXT: asrs r4, r0, #31
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r1, s6
-; CHECK-NEXT: umull r2, r5, r0, r3
+; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: umull lr, r12, r0, r1
+; CHECK-NEXT: umull r2, r5, r0, r3
; CHECK-NEXT: vmov q1[2], q1[0], r2, lr
; CHECK-NEXT: asrs r2, r1, #31
; CHECK-NEXT: mla r2, r0, r2, r12
@@ -528,13 +524,11 @@ define arm_aapcs_vfpcc <4 x i64> @zext32_0213_ext0(<8 x i32> %src1, i32 %src2) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov.f32 s0, s1
-; CHECK-NEXT: vmov.f32 s2, s3
; CHECK-NEXT: umull r1, r12, r1, r0
; CHECK-NEXT: umull r3, r2, r3, r0
; CHECK-NEXT: vmov q2[2], q2[0], r3, r1
-; CHECK-NEXT: vmov r1, s2
-; CHECK-NEXT: vmov r3, s0
+; CHECK-NEXT: vmov r1, s3
+; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: vmov q2[3], q2[1], r2, r12
; CHECK-NEXT: vmov q0, q2
; CHECK-NEXT: umull r1, r2, r1, r0
@@ -557,13 +551,11 @@ define arm_aapcs_vfpcc <4 x i64> @zext32_ext0_0213(<8 x i32> %src1, i32 %src2) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov.f32 s0, s1
-; CHECK-NEXT: vmov.f32 s2, s3
; CHECK-NEXT: umull r1, r12, r0, r1
; CHECK-NEXT: umull r3, r2, r0, r3
; CHECK-NEXT: vmov q2[2], q2[0], r3, r1
-; CHECK-NEXT: vmov r1, s2
-; CHECK-NEXT: vmov r3, s0
+; CHECK-NEXT: vmov r1, s3
+; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: vmov q2[3], q2[1], r2, r12
; CHECK-NEXT: vmov q0, q2
; CHECK-NEXT: umull r1, r2, r0, r1
diff --git a/llvm/test/CodeGen/X86/peephole-reg-sequence.mir b/llvm/test/CodeGen/X86/peephole-reg-sequence.mir
new file mode 100644
index 0000000000000..52b05a0d2b4c2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/peephole-reg-sequence.mir
@@ -0,0 +1,28 @@
+# RUN: llc -mtriple=x86_64-- -mcpu=skylake-avx512 -run-pass=peephole-opt -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: regseq_source_class_differs_from_subreg_use
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: regseq_source_class_differs_from_subreg_use
+ ; CHECK: [[SRC:%[0-9]+]]:vr128 = AVX512_128_SET0
+ ; CHECK-NEXT: [[USE:%[0-9]+]]:vr128 = MOVAPSrr [[SRC]]
+ ; CHECK-NEXT: RET 0
+ %0:vr128x = AVX512_128_SET0
+ %1:vr256 = REG_SEQUENCE %0, %subreg.sub_xmm
+ ; Must constrain %0 from vr128x to vr128 for the fold.
+ %2:vr128 = MOVAPSrr %1.sub_xmm
+ RET 0
+
+...
+---
+name: regseq_fold_preserves_source_subreg
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: regseq_fold_preserves_source_subreg
+ %0:vr256 = AVX_SET0
+ %1:vr256 = REG_SEQUENCE %0.sub_xmm, %subreg.sub_xmm
+ %2:vr128 = MOVAPSrr %1.sub_xmm
+ RET 0
+
+...
>From ecd9c0fac2abfa3261c2f9c07f6173a35f5260b4 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 24 Jun 2026 08:55:12 -0400
Subject: [PATCH 2/6] [PeepholeOptimizer] Run foldRegSequenceUses after
optimizeCoalescableCopy
Allow foldRegSequenceUses to run immediately after optimizeCoalescableCopy
for REG_SEQUENCE instructions. This ensures sources are optimized before
REG_SEQUENCE use folding, enabling better code generation.
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 20 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 72 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 30 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 30 +-
llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll | 44 +-
...e92561-restore-undef-scc-verifier-error.ll | 56 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 22 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 6 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 30 +-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 24 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 36 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f64.ll | 12 +-
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 36 +-
...mdgcn.struct.ptr.buffer.atomic.fmin.f64.ll | 12 +-
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 15 +-
llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 476 ++--
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 336 +--
.../AMDGPU/memset-param-combinations.ll | 227 +-
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 176 +-
.../splitkit-getsubrangeformask-phi-extend.ll | 2055 ++++++++---------
...r-descriptor-waterfall-loop-idom-update.ll | 19 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 18 +-
llvm/test/CodeGen/Thumb2/mve-clmul.ll | 8 +-
.../Thumb2/mve-laneinterleaving-cost.ll | 64 +-
.../CodeGen/Thumb2/mve-laneinterleaving.ll | 25 +-
llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll | 10 +-
llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll | 23 +-
llvm/test/CodeGen/Thumb2/mve-shuffle.ll | 60 +-
llvm/test/CodeGen/Thumb2/mve-vabdus.ll | 4 +-
llvm/test/CodeGen/Thumb2/mve-vld3.ll | 137 +-
35 files changed, 1970 insertions(+), 2153 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index f303fb2e3b6c1..d9c3823be5e32 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -2003,7 +2003,10 @@ bool PeepholeOptimizer::run(MachineFunction &MF) {
if (isCoalescableCopy(*MI) && optimizeCoalescableCopy(*MI)) {
// MI is just rewritten.
Changed = true;
- continue;
+
+ if (!MI->isRegSequence())
+ continue;
+ // Allow REG_SEQUENCE use folding to run.
}
if (MI->isRegSequence()) {
@@ -2014,8 +2017,9 @@ bool PeepholeOptimizer::run(MachineFunction &MF) {
MI->eraseFromParent();
}
Changed = true;
- continue;
}
+ // Always continue. No further REG_SEQUENCE opts.
+ continue;
}
if (MI->isCopy() && (foldRedundantCopy(*MI) ||
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
index 1791bf59af12b..c5d210d13bd7d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
@@ -413,7 +413,7 @@ define <10 x i16> @bitcast_v5i32_to_v10i16(<5 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
@@ -1003,7 +1003,7 @@ define <10 x half> @bitcast_v5i32_to_v10f16(<5 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
@@ -1632,7 +1632,7 @@ define <10 x i16> @bitcast_v5f32_to_v10i16(<5 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
@@ -2237,7 +2237,7 @@ define <10 x half> @bitcast_v5f32_to_v10f16(<5 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v5, v0, v4, 16
+; SI-NEXT: v_alignbit_b32 v5, s4, v4, 16
; SI-NEXT: v_alignbit_b32 v6, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v7, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
index 609d14e386544..ba83ef3f2d8f6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
@@ -463,7 +463,7 @@ define <14 x i16> @bitcast_v7i32_to_v14i16(<7 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
@@ -1175,7 +1175,7 @@ define <14 x half> @bitcast_v7i32_to_v14f16(<7 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
@@ -1940,7 +1940,7 @@ define <14 x i16> @bitcast_v7f32_to_v14i16(<7 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
@@ -2664,7 +2664,7 @@ define <14 x half> @bitcast_v7f32_to_v14f16(<7 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v7, v0, v6, 16
+; SI-NEXT: v_alignbit_b32 v7, s4, v6, 16
; SI-NEXT: v_alignbit_b32 v8, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v9, v3, v2, 16
; SI-NEXT: v_alignbit_b32 v11, v1, v0, 16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
index 59969f18bb145..1a4a8abbe3d5d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
@@ -511,7 +511,7 @@ define <18 x i16> @bitcast_v9i32_to_v18i16(<9 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_alignbit_b32 v10, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
@@ -1342,7 +1342,7 @@ define <18 x half> @bitcast_v9i32_to_v18f16(<9 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_alignbit_b32 v10, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
@@ -2241,7 +2241,7 @@ define <18 x i16> @bitcast_v9f32_to_v18i16(<9 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_alignbit_b32 v10, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
@@ -3101,7 +3101,7 @@ define <18 x half> @bitcast_v9f32_to_v18f16(<9 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 16
; SI-NEXT: v_alignbit_b32 v10, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v11, v5, v4, 16
; SI-NEXT: v_alignbit_b32 v12, v3, v2, 16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
index 2500fbac40aa0..4502aabbb4fd6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
@@ -565,7 +565,7 @@ define <22 x i16> @bitcast_v11i32_to_v22i16(<11 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB4_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_alignbit_b32 v11, v9, v8, 16
; SI-NEXT: v_alignbit_b32 v13, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
@@ -1518,7 +1518,7 @@ define <22 x half> @bitcast_v11i32_to_v22f16(<11 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB8_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_alignbit_b32 v11, v9, v8, 16
; SI-NEXT: v_alignbit_b32 v13, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
@@ -2552,7 +2552,7 @@ define <22 x i16> @bitcast_v11f32_to_v22i16(<11 x float> %a, i32 %b) #0 {
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB12_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_alignbit_b32 v11, v9, v8, 16
; SI-NEXT: v_alignbit_b32 v13, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
@@ -3530,7 +3530,7 @@ define <22 x half> @bitcast_v11f32_to_v22f16(<11 x float> %a, i32 %b) #0 {
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB16_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v12, v0, v10, 16
+; SI-NEXT: v_alignbit_b32 v12, s4, v10, 16
; SI-NEXT: v_alignbit_b32 v11, v9, v8, 16
; SI-NEXT: v_alignbit_b32 v13, v7, v6, 16
; SI-NEXT: v_alignbit_b32 v14, v5, v4, 16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 1be677ad46306..ec6bace33a4e2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -380,9 +380,9 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB4_3: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v11, v0, v8, 24
-; SI-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 8
+; SI-NEXT: v_alignbit_b32 v11, s4, v8, 24
+; SI-NEXT: v_alignbit_b32 v10, s4, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 8
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
@@ -2546,7 +2546,7 @@ define <6 x half> @bitcast_v3i32_to_v6f16(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
@@ -3041,7 +3041,7 @@ define <6 x i16> @bitcast_v3i32_to_v6i16(<3 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
@@ -3525,9 +3525,9 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB20_3: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v11, v0, v8, 24
-; SI-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v9, v0, v8, 8
+; SI-NEXT: v_alignbit_b32 v11, s4, v8, 24
+; SI-NEXT: v_alignbit_b32 v10, s4, v8, 16
+; SI-NEXT: v_alignbit_b32 v9, s4, v8, 8
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
@@ -5720,7 +5720,7 @@ define <6 x half> @bitcast_v3f32_to_v6f16(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
@@ -6223,7 +6223,7 @@ define <6 x i16> @bitcast_v3f32_to_v6i16(<3 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v3, s4, v2, 16
; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 7df50850f6357..60899b4be4405 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -388,8 +388,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -417,8 +418,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 sc0
-; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr4
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2:
@@ -445,8 +447,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -531,8 +534,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_add_f32 v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
; GFX90A-NEXT: ; %bb.2:
@@ -2388,8 +2392,9 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
-; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr4
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2:
@@ -2545,8 +2550,9 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2:
@@ -7327,8 +7333,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -7356,8 +7363,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], 0 offen offset:1024 sc0
-; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr4
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2:
@@ -7502,8 +7510,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
@@ -9646,8 +9655,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -10196,7 +10206,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -10207,32 +10217,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
+; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX7-NEXT: .LBB28_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
-; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v10
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v7, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_add_f32_e32 v8, v8, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; GFX7-NEXT: v_alignbit_b32 v5, v8, v5, 16
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v7, v5
@@ -10254,17 +10264,17 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB28_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v7
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 08e1a59245fd6..04d5ae50e67aa 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -379,8 +379,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -467,8 +468,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -495,8 +497,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -695,8 +698,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB2_1
; GFX7-NEXT: ; %bb.2:
@@ -721,8 +725,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr0
; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB2_1
; GFX6-NEXT: ; %bb.2:
@@ -1620,8 +1625,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
-; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr4
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2:
@@ -1713,8 +1719,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1743,8 +1750,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_max_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2:
@@ -1898,8 +1906,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
; GFX7-NEXT: ; %bb.2:
@@ -1925,8 +1934,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr0
; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index be032802e75d1..d768ded12222d 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -379,8 +379,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -467,8 +468,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -495,8 +497,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -695,8 +698,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB2_1
; GFX7-NEXT: ; %bb.2:
@@ -721,8 +725,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr0
; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB2_1
; GFX6-NEXT: ; %bb.2:
@@ -1620,8 +1625,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
-; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr4
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2:
@@ -1713,8 +1719,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1743,8 +1750,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
-; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr4
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2:
@@ -1898,8 +1906,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: ; implicit-def: $vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
; GFX7-NEXT: ; %bb.2:
@@ -1925,8 +1934,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
-; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX6-NEXT: ; implicit-def: $vgpr0
; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
index 7a576b1c58c55..dd34812c29cd1 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
@@ -163,33 +163,33 @@ define amdgpu_kernel void @test_copy_v4i8_x3(ptr addrspace(1) %out0, ptr addrspa
define amdgpu_kernel void @test_copy_v4i8_x4(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3, ptr addrspace(1) %in) nounwind {
; SI-LABEL: test_copy_v4i8_x4:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x11
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s10, 0
-; SI-NEXT: s_mov_b32 s11, s3
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x11
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_mov_b32 s3, s11
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; SI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s14, s2
-; SI-NEXT: s_mov_b32 s15, s3
-; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s18, s10
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
-; SI-NEXT: s_mov_b32 s19, s3
-; SI-NEXT: s_mov_b32 s22, s2
-; SI-NEXT: s_mov_b32 s23, s3
-; SI-NEXT: s_mov_b32 s12, s6
-; SI-NEXT: s_mov_b32 s13, s7
-; SI-NEXT: s_mov_b32 s16, s8
-; SI-NEXT: s_mov_b32 s17, s9
-; SI-NEXT: s_mov_b32 s20, s10
-; SI-NEXT: s_mov_b32 s21, s11
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_mov_b32 s22, s10
+; SI-NEXT: s_mov_b32 s23, s11
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s20, s6
+; SI-NEXT: s_mov_b32 s21, s7
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: buffer_store_dword v0, off, s[12:15], 0
; SI-NEXT: buffer_store_dword v0, off, s[16:19], 0
; SI-NEXT: buffer_store_dword v0, off, s[20:23], 0
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index 4408eb79ae671..2cbdbc1368e31 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -12,9 +12,9 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG: ; %bb.0: ; %bb
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: s_clause 0x1
-; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:16
-; SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
-; SDAG-NEXT: v_mov_b32_e32 v8, 0
+; SDAG-NEXT: global_load_b128 v[2:5], v[0:1], off offset:16
+; SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: s_mov_b32 s8, 0
; SDAG-NEXT: s_mov_b32 s12, exec_lo
; SDAG-NEXT: s_mov_b32 s9, s8
@@ -23,26 +23,27 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_mov_b32 s13, s12
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_readfirstlane_b32 s0, v0
-; SDAG-NEXT: v_readfirstlane_b32 s1, v1
-; SDAG-NEXT: v_readfirstlane_b32 s2, v2
-; SDAG-NEXT: v_readfirstlane_b32 s3, v3
-; SDAG-NEXT: v_readfirstlane_b32 s4, v4
-; SDAG-NEXT: v_readfirstlane_b32 s5, v5
-; SDAG-NEXT: v_readfirstlane_b32 s6, v6
-; SDAG-NEXT: v_readfirstlane_b32 s7, v7
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; SDAG-NEXT: v_readfirstlane_b32 s2, v8
+; SDAG-NEXT: v_readfirstlane_b32 s3, v9
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: v_readfirstlane_b32 s5, v3
+; SDAG-NEXT: v_readfirstlane_b32 s6, v4
+; SDAG-NEXT: v_readfirstlane_b32 s7, v5
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[6:7]
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[8:9]
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
+; SDAG-NEXT: image_sample_c_lz v0, [v1, v1, v1, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
-; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7_vgpr8_vgpr9
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; SDAG-NEXT: s_cbranch_execnz .LBB0_1
; SDAG-NEXT: ; %bb.2:
; SDAG-NEXT: s_mov_b32 exec_lo, s12
-; SDAG-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 1.0
+; SDAG-NEXT: v_dual_mov_b32 v2, 0x7fc00000 :: v_dual_mov_b32 v3, 1.0
; SDAG-NEXT: s_mov_b32 s0, s8
; SDAG-NEXT: s_mov_b32 s1, s8
; SDAG-NEXT: s_mov_b32 s2, s8
@@ -52,18 +53,19 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_mov_b32 s6, s8
; SDAG-NEXT: s_mov_b32 s7, s8
; SDAG-NEXT: s_clause 0x2
-; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v2, [v1, v1, v2, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v4, [v1, v1, v1, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v3, [v1, v3, v1, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_waitcnt vmcnt(2)
-; SDAG-NEXT: v_dual_add_f32 v0, v9, v0 :: v_dual_mov_b32 v9, v8
+; SDAG-NEXT: v_add_f32_e32 v0, v0, v2
+; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-NEXT: v_add_f32_e32 v0, v1, v0
-; SDAG-NEXT: v_add_f32_e32 v0, v2, v0
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_dual_add_f32 v0, v3, v0 :: v_dual_mov_b32 v3, 0
+; SDAG-NEXT: v_add_f32_e32 v0, v4, v0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_dual_mul_f32 v7, 0x3e800000, v0 :: v_dual_mov_b32 v0, 0
-; SDAG-NEXT: image_store v[7:9], [v0, v0], s[0:7] dim:SQ_RSRC_IMG_2D unorm
+; SDAG-NEXT: v_mul_f32_e32 v0, 0x3e800000, v0
+; SDAG-NEXT: image_store v[0:2], [v3, v3], s[0:7] dim:SQ_RSRC_IMG_2D unorm
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: issue92561:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index cc3bd9706887e..dc96087704cbc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -121,9 +121,10 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr6
; GFX1200-NEXT: ; implicit-def: $vgpr5
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -135,30 +136,31 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v11, v4 :: v_dual_mov_b32 v10, v3
-; GFX1250-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v8, v1
+; GFX1250-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
+; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v5
; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_readfirstlane_b32 s0, v8
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v9
-; GFX1250-NEXT: v_readfirstlane_b32 s2, v10
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v11
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v9
; GFX1250-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[8:9]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[10:11]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[8:9]
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr6
; GFX1250-NEXT: ; implicit-def: $vgpr1
+; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 7599cde4eb406..50312c14ab15f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -57,9 +57,10 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB2_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -94,10 +95,11 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB3_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index 4723cabc315e5..c0e84ae4958f7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -209,10 +209,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[8:11], s12 idxen offen
-; GFX908-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr7
; GFX908-NEXT: ; implicit-def: $vgpr0
; GFX908-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2:
@@ -244,10 +245,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[8:11], s12 idxen offen
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2
; GFX90A-NEXT: ; implicit-def: $vgpr7
; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2:
@@ -278,10 +280,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s8 idxen offen
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2
; GFX942-NEXT: ; implicit-def: $vgpr7
; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2:
@@ -312,10 +315,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -347,10 +351,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -379,10 +384,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[8:11], s12 idxen offen
-; GFX908-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr7
; GFX908-NEXT: ; implicit-def: $vgpr0
; GFX908-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2:
@@ -414,10 +420,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[8:11], s12 idxen offen
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2
; GFX90A-NEXT: ; implicit-def: $vgpr7
; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2:
@@ -448,10 +455,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s8 idxen offen
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2
; GFX942-NEXT: ; implicit-def: $vgpr7
; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2:
@@ -482,10 +490,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -517,10 +526,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index ec811326b4cb7..f90d4a067b63c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -194,9 +194,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[8:11], s12 idxen offen glc
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2
; GFX90A-NEXT: ; implicit-def: $vgpr7
; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2:
@@ -228,9 +229,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s8 idxen offen sc0
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2
; GFX942-NEXT: ; implicit-def: $vgpr7
; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2:
@@ -262,9 +264,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -298,9 +301,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -336,9 +340,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[8:11], s12 idxen offen glc
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2
; GFX90A-NEXT: ; implicit-def: $vgpr7
; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2:
@@ -370,9 +375,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s8 idxen offen sc0
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2
; GFX942-NEXT: ; implicit-def: $vgpr7
; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2:
@@ -404,9 +410,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -440,9 +447,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index b5b3790c90f69..8e314c15e5291 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -425,8 +425,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX6-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2:
@@ -449,8 +450,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX7-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2:
@@ -474,8 +476,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -500,8 +503,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -530,8 +534,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -562,8 +567,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -594,9 +600,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s12 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: ; implicit-def: $vgpr7
; GFX6-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX6-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB9_1
; GFX6-NEXT: ; %bb.2:
@@ -622,9 +629,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s12 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: ; implicit-def: $vgpr7
; GFX7-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX7-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2:
@@ -650,9 +658,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -680,9 +689,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -713,9 +723,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -748,9 +759,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64.ll
index eb0ac4ecac68c..40ac71703e48c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64.ll
@@ -169,8 +169,9 @@ define double @struct_ptr_buffer_atomic_fmax_f64_ret__vgpr_val__vgpr_rsrc__vgpr_
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v[6:7], s[8:11], s16 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX6-NEXT: ; implicit-def: $vgpr2
; GFX6-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX6-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2:
@@ -193,8 +194,9 @@ define double @struct_ptr_buffer_atomic_fmax_f64_ret__vgpr_val__vgpr_rsrc__vgpr_
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v[6:7], s[8:11], s16 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2:
@@ -226,9 +228,10 @@ define double @struct_ptr_buffer_atomic_fmax_f64_ret__vgpr_val__sgpr_rsrc__vgpr_
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmax_x2 v[0:1], v[6:7], s[8:11], s12 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX6-NEXT: ; implicit-def: $vgpr2
; GFX6-NEXT: ; implicit-def: $vgpr8
; GFX6-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX6-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB9_1
; GFX6-NEXT: ; %bb.2:
@@ -254,9 +257,10 @@ define double @struct_ptr_buffer_atomic_fmax_f64_ret__vgpr_val__sgpr_rsrc__vgpr_
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v[6:7], s[8:11], s12 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: ; implicit-def: $vgpr8
; GFX7-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index e19c57c3bd0db..42a73002dcd90 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -425,8 +425,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX6-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2:
@@ -449,8 +450,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX7-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2:
@@ -474,8 +476,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -500,8 +503,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -530,8 +534,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -562,8 +567,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -594,9 +600,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s12 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: ; implicit-def: $vgpr7
; GFX6-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX6-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB9_1
; GFX6-NEXT: ; %bb.2:
@@ -622,9 +629,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s12 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: ; implicit-def: $vgpr7
; GFX7-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX7-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2:
@@ -650,9 +658,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -680,9 +689,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -713,9 +723,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT: ; implicit-def: $vgpr1
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -748,9 +759,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr2
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64.ll
index 92c717c7d68d9..f849310e22a84 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64.ll
@@ -169,8 +169,9 @@ define double @struct_ptr_buffer_atomic_fmin_f64_ret__vgpr_val__vgpr_rsrc__vgpr_
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v[6:7], s[8:11], s16 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX6-NEXT: ; implicit-def: $vgpr2
; GFX6-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX6-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2:
@@ -193,8 +194,9 @@ define double @struct_ptr_buffer_atomic_fmin_f64_ret__vgpr_val__vgpr_rsrc__vgpr_
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v[6:7], s[8:11], s16 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2:
@@ -226,9 +228,10 @@ define double @struct_ptr_buffer_atomic_fmin_f64_ret__vgpr_val__sgpr_rsrc__vgpr_
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v[6:7], s[8:11], s12 idxen offen offset:256 glc
-; GFX6-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX6-NEXT: ; implicit-def: $vgpr2
; GFX6-NEXT: ; implicit-def: $vgpr8
; GFX6-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX6-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB9_1
; GFX6-NEXT: ; %bb.2:
@@ -254,9 +257,10 @@ define double @struct_ptr_buffer_atomic_fmin_f64_ret__vgpr_val__sgpr_rsrc__vgpr_
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v[6:7], s[8:11], s12 idxen offen offset:256 glc
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: ; implicit-def: $vgpr8
; GFX7-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index dc4797ce8bc45..ea2de07d3e46d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -21,8 +21,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -48,8 +49,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr4
+; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-NEXT: ; %bb.2:
@@ -74,8 +76,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX8-NEXT: s_nop 0
; GFX8-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: ; implicit-def: $vgpr4
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
; GFX8-NEXT: ; %bb.2:
@@ -102,8 +105,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
@@ -128,8 +132,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index 734bcfdd4378a..dae5bc2afd8b7 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -8010,105 +8010,105 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s1
+; SI-NEXT: v_mov_b32_e32 v8, s1
; SI-NEXT: s_mov_b32 m0, -1
-; SI-NEXT: ds_read2_b64 v[4:7], v12 offset0:2 offset1:3
-; SI-NEXT: ds_read2_b64 v[0:3], v12 offset1:1
-; SI-NEXT: ds_read2_b64 v[8:11], v12 offset0:6 offset1:7
-; SI-NEXT: ds_read2_b64 v[12:15], v12 offset0:4 offset1:5
+; SI-NEXT: ds_read2_b64 v[12:15], v8 offset0:2 offset1:3
+; SI-NEXT: ds_read2_b64 v[0:3], v8 offset1:1
+; SI-NEXT: ds_read2_b64 v[4:7], v8 offset0:6 offset1:7
+; SI-NEXT: ds_read2_b64 v[8:11], v8 offset0:4 offset1:5
; SI-NEXT: s_waitcnt lgkmcnt(3)
-; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v7
-; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v7
-; SI-NEXT: v_bfe_i32 v18, v7, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v15
+; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v15
+; SI-NEXT: v_bfe_i32 v18, v15, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: v_mov_b32_e32 v7, s0
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:14 offset1:15
-; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v5
-; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v5
-; SI-NEXT: v_bfe_i32 v18, v5, 0, 16
+; SI-NEXT: v_mov_b32_e32 v15, s0
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:14 offset1:15
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v13
+; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v13
+; SI-NEXT: v_bfe_i32 v18, v13, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:10 offset1:11
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:10 offset1:11
; SI-NEXT: s_waitcnt lgkmcnt(4)
; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v3
; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v3
; SI-NEXT: v_bfe_i32 v18, v3, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:6 offset1:7
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:6 offset1:7
; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v1
; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v1
; SI-NEXT: v_bfe_i32 v18, v1, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:2 offset1:3
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:2 offset1:3
; SI-NEXT: s_waitcnt lgkmcnt(5)
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v7
+; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v7
+; SI-NEXT: v_bfe_i32 v18, v7, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:30 offset1:31
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v5
+; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v5
+; SI-NEXT: v_bfe_i32 v18, v5, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:26 offset1:27
+; SI-NEXT: s_waitcnt lgkmcnt(6)
; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v11
; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v11
; SI-NEXT: v_bfe_i32 v18, v11, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:30 offset1:31
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:22 offset1:23
; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v9
; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v9
; SI-NEXT: v_bfe_i32 v18, v9, 0, 16
; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:26 offset1:27
-; SI-NEXT: s_waitcnt lgkmcnt(6)
-; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v15
-; SI-NEXT: v_ashrrev_i32_e32 v16, 16, v15
-; SI-NEXT: v_bfe_i32 v18, v15, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; SI-NEXT: ds_write2_b64 v7, v[18:19], v[16:17] offset0:22 offset1:23
-; SI-NEXT: v_ashrrev_i32_e32 v16, 31, v13
-; SI-NEXT: v_ashrrev_i32_e32 v15, 16, v13
-; SI-NEXT: v_bfe_i32 v17, v13, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; SI-NEXT: ds_write2_b64 v7, v[17:18], v[15:16] offset0:18 offset1:19
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT: v_bfe_i32 v5, v6, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; SI-NEXT: v_bfe_i32 v15, v1, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; SI-NEXT: ds_write2_b64 v7, v[5:6], v[15:16] offset0:12 offset1:13
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v10
-; SI-NEXT: v_bfe_i32 v3, v4, 0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v8
-; SI-NEXT: v_bfe_i32 v5, v1, 0, 16
+; SI-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:18 offset1:19
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
+; SI-NEXT: v_bfe_i32 v13, v14, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; SI-NEXT: v_bfe_i32 v16, v1, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; SI-NEXT: ds_write2_b64 v15, v[13:14], v[16:17] offset0:12 offset1:13
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v6
+; SI-NEXT: v_bfe_i32 v11, v12, 0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
+; SI-NEXT: v_bfe_i32 v13, v1, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; SI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; SI-NEXT: ds_write2_b64 v15, v[11:12], v[13:14] offset0:8 offset1:9
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v8
+; SI-NEXT: v_bfe_i32 v1, v8, 0, 16
+; SI-NEXT: v_bfe_i32 v3, v10, 0, 16
+; SI-NEXT: v_bfe_i32 v5, v4, 0, 16
+; SI-NEXT: v_bfe_i32 v7, v6, 0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_bfe_i32 v8, v0, 0, 16
+; SI-NEXT: v_bfe_i32 v9, v2, 0, 16
+; SI-NEXT: v_bfe_i32 v11, v16, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; SI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; SI-NEXT: ds_write2_b64 v15, v[9:10], v[11:12] offset0:4 offset1:5
+; SI-NEXT: v_bfe_i32 v10, v14, 0, 16
+; SI-NEXT: v_bfe_i32 v12, v13, 0, 16
+; SI-NEXT: v_bfe_i32 v16, v18, 0, 16
+; SI-NEXT: v_bfe_i32 v13, v4, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; SI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; SI-NEXT: ds_write2_b64 v15, v[8:9], v[13:14] offset1:1
+; SI-NEXT: v_bfe_i32 v18, v17, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v2, 31, v1
; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; SI-NEXT: ds_write2_b64 v7, v[3:4], v[5:6] offset0:8 offset1:9
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; SI-NEXT: v_bfe_i32 v1, v12, 0, 16
-; SI-NEXT: v_bfe_i32 v3, v14, 0, 16
-; SI-NEXT: v_bfe_i32 v5, v8, 0, 16
-; SI-NEXT: v_bfe_i32 v8, v10, 0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; SI-NEXT: v_bfe_i32 v9, v0, 0, 16
-; SI-NEXT: v_bfe_i32 v10, v2, 0, 16
-; SI-NEXT: v_bfe_i32 v12, v11, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v8, 31, v7
; SI-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; SI-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; SI-NEXT: ds_write2_b64 v7, v[10:11], v[12:13] offset0:4 offset1:5
-; SI-NEXT: v_bfe_i32 v11, v6, 0, 16
-; SI-NEXT: v_bfe_i32 v13, v4, 0, 16
-; SI-NEXT: v_bfe_i32 v15, v15, 0, 16
-; SI-NEXT: v_bfe_i32 v16, v14, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; SI-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; SI-NEXT: ds_write2_b64 v7, v[9:10], v[16:17] offset1:1
-; SI-NEXT: v_bfe_i32 v17, v18, 0, 16
-; SI-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; SI-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; SI-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; SI-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; SI-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; SI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; SI-NEXT: ds_write2_b64 v7, v[8:9], v[17:18] offset0:28 offset1:29
-; SI-NEXT: ds_write2_b64 v7, v[5:6], v[15:16] offset0:24 offset1:25
-; SI-NEXT: ds_write2_b64 v7, v[3:4], v[13:14] offset0:20 offset1:21
-; SI-NEXT: ds_write2_b64 v7, v[1:2], v[11:12] offset0:16 offset1:17
+; SI-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; SI-NEXT: ds_write2_b64 v15, v[7:8], v[18:19] offset0:28 offset1:29
+; SI-NEXT: ds_write2_b64 v15, v[5:6], v[16:17] offset0:24 offset1:25
+; SI-NEXT: ds_write2_b64 v15, v[3:4], v[12:13] offset0:20 offset1:21
+; SI-NEXT: ds_write2_b64 v15, v[1:2], v[10:11] offset0:16 offset1:17
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_sextload_v32i16_to_v32i64:
@@ -8682,111 +8682,111 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v4, s1
-; VI-DS128-NEXT: ds_read_b128 v[0:3], v4 offset:48
-; VI-DS128-NEXT: ds_read_b128 v[9:12], v4 offset:32
-; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
-; VI-DS128-NEXT: ds_read_b128 v[17:20], v4 offset:16
-; VI-DS128-NEXT: ds_read_b128 v[4:7], v4
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-DS128-NEXT: v_bfe_i32 v13, v2, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
+; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
+; VI-DS128-NEXT: ds_read_b128 v[5:8], v0 offset:48
+; VI-DS128-NEXT: ds_read_b128 v[9:12], v0 offset:32
+; VI-DS128-NEXT: v_mov_b32_e32 v4, s0
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v15, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:224
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; VI-DS128-NEXT: ds_read_b128 v[17:20], v0 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[0:3], v0
+; VI-DS128-NEXT: ds_write_b128 v4, v[13:16] offset:224
+; VI-DS128-NEXT: v_bfe_i32 v15, v7, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v13, v8, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:240
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; VI-DS128-NEXT: v_bfe_i32 v13, v0, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:208
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; VI-DS128-NEXT: ds_write_b128 v4, v[13:16] offset:240
+; VI-DS128-NEXT: v_bfe_i32 v15, v7, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; VI-DS128-NEXT: v_bfe_i32 v13, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v5, v6, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; VI-DS128-NEXT: ds_write_b128 v4, v[5:8] offset:208
; VI-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; VI-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:160
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v12
-; VI-DS128-NEXT: v_bfe_i32 v0, v12, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:176
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; VI-DS128-NEXT: v_bfe_i32 v0, v9, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:128
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v11
+; VI-DS128-NEXT: v_bfe_i32 v5, v11, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; VI-DS128-NEXT: ds_write_b128 v4, v[5:8] offset:160
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v12
+; VI-DS128-NEXT: v_bfe_i32 v5, v12, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; VI-DS128-NEXT: ds_write_b128 v4, v[5:8] offset:176
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v9
+; VI-DS128-NEXT: v_bfe_i32 v5, v9, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; VI-DS128-NEXT: ds_write_b128 v4, v[5:8] offset:128
; VI-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; VI-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v10
+; VI-DS128-NEXT: v_bfe_i32 v5, v1, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; VI-DS128-NEXT: v_bfe_i32 v7, v1, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v10
; VI-DS128-NEXT: v_bfe_i32 v9, v10, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v11, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v19
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:144
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v19
+; VI-DS128-NEXT: ds_write_b128 v4, v[9:12] offset:144
; VI-DS128-NEXT: v_bfe_i32 v9, v19, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v11, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v20
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:96
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v20
+; VI-DS128-NEXT: ds_write_b128 v4, v[9:12] offset:96
; VI-DS128-NEXT: v_bfe_i32 v9, v20, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v11, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v17
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:112
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; VI-DS128-NEXT: ds_write_b128 v4, v[9:12] offset:112
; VI-DS128-NEXT: v_bfe_i32 v9, v17, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v11, v1, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:64
-; VI-DS128-NEXT: v_bfe_i32 v9, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; VI-DS128-NEXT: ds_write_b128 v4, v[9:12] offset:64
+; VI-DS128-NEXT: v_bfe_i32 v9, v0, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: v_bfe_i32 v11, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v18
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:192
+; VI-DS128-NEXT: v_bfe_i32 v11, v0, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
+; VI-DS128-NEXT: ds_write_b128 v4, v[13:16] offset:192
; VI-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:80
-; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: ds_write_b128 v4, v[13:16] offset:80
+; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; VI-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:32
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:48
-; VI-DS128-NEXT: ds_write_b128 v8, v[9:12]
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:16
+; VI-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:32
+; VI-DS128-NEXT: ds_write_b128 v4, v[13:16] offset:48
+; VI-DS128-NEXT: ds_write_b128 v4, v[9:12]
+; VI-DS128-NEXT: ds_write_b128 v4, v[5:8] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_sextload_v32i16_to_v32i64:
@@ -8794,111 +8794,111 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DS128-NEXT: v_mov_b32_e32 v17, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v17 offset:48
+; GFX9-DS128-NEXT: ds_read_b128 v[9:12], v17 offset:48
; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v17 offset:32
-; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v17
-; GFX9-DS128-NEXT: ds_read_b128 v[17:20], v17 offset:16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v8, s0
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v11
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:224
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v12
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v17
+; GFX9-DS128-NEXT: ds_read_b128 v[17:20], v17 offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:224
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v12, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:240
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v4, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v5, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:208
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v2, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:240
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v9
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v9, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:192
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v10, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:208
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(6)
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v2, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:160
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v3, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:160
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v3, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:176
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v0, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:176
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:128
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:192
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v1, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v0, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:144
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v19, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v20
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v20, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:112
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v17, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:64
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:128
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:144
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(8)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v19
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v19, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:96
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v20
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v20, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:112
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v17
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v17, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:64
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v18
; GFX9-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:80
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-DS128-NEXT: v_bfe_i32 v17, v10, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v19, v0, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v9, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:80
+; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v5, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[17:20] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[6:9]
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3]
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = sext <32 x i16> %load to <32 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index 7786d47810b28..dad7ded946beb 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -1043,55 +1043,55 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p1_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB7_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[10:13], v9
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: ds_read_b128 v[9:12], v8
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_2
; CHECK-NEXT: .LBB7_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB7_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; CHECK-NEXT: .LBB7_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_u8 v7, v2
+; CHECK-NEXT: ds_read_u8 v5, v2
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_5
; CHECK-NEXT: .LBB7_6: ; %Flow7
@@ -1252,60 +1252,60 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p1_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB9_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB9_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: buffer_load_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_2
; CHECK-NEXT: .LBB9_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB9_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; CHECK-NEXT: .LBB9_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_5
; CHECK-NEXT: .LBB9_6: ; %Flow7
@@ -1465,39 +1465,39 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p3_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB11_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB11_2
; CHECK-NEXT: .LBB11_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB11_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1508,7 +1508,7 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1529,13 +1529,13 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p3_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v2
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
; CHECK-NEXT: v_cmpx_ge_u32_e32 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_3
@@ -1550,16 +1550,16 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB12_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .LBB12_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[9:12], v3
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: ds_read_b128 v[9:12], v7
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b128 v8, v[9:12]
@@ -1578,10 +1578,10 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: .LBB12_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v2, v1
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b8 v0, v2
@@ -1593,7 +1593,8 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB12_2
; CHECK-NEXT: .LBB12_10: ; %memmove_copy_backwards
@@ -1602,19 +1603,19 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB12_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v8, v7
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b8 v6, v8
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: ds_write_b8 v3, v8
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -1, v3
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB12_12
; CHECK-NEXT: .LBB12_13: ; %Flow45
@@ -1624,21 +1625,21 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v5
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v3
; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
-; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v1, v3
+; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v4, vcc_lo
; CHECK-NEXT: .LBB12_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[5:8], v4
+; CHECK-NEXT: ds_read_b128 v[4:7], v3
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v3
; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; CHECK-NEXT: s_or_b32 s6, vcc_lo, s6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b128 v2, v[5:8]
+; CHECK-NEXT: ds_write_b128 v2, v[4:7]
; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_15
@@ -1656,39 +1657,39 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p3_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB13_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB13_2
; CHECK-NEXT: .LBB13_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB13_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1699,7 +1700,7 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1936,32 +1937,32 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p5_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB16_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB16_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB16_2
@@ -1969,10 +1970,10 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB16_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1983,7 +1984,7 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2068,32 +2069,32 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p5_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB18_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB18_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB18_2
@@ -2101,10 +2102,10 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB18_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -2115,7 +2116,7 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2135,13 +2136,13 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p5_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v2
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
; CHECK-NEXT: v_cmpx_ge_u32_e32 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB19_3
@@ -2155,20 +2156,20 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB19_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v9, v3, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v10, v3, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v11, v3, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v3, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
+; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v12, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
@@ -2177,7 +2178,7 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -2194,10 +2195,10 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: .LBB19_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen
@@ -2209,7 +2210,8 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB19_2
; CHECK-NEXT: .LBB19_10: ; %memmove_copy_backwards
@@ -2218,19 +2220,19 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB19_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v8, v7, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_byte v8, v6, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: buffer_store_byte v8, v3, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -1, v3
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB19_12
; CHECK-NEXT: .LBB19_13: ; %Flow45
@@ -2240,24 +2242,24 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v5
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v3
; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
-; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v1, v3
+; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v4, vcc_lo
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v3, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v5, v3, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v6, v3, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v7, v3, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v3
; CHECK-NEXT: s_waitcnt vmcnt(3)
-; CHECK-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 16d50bf927f85..3cce4d4e6faa5 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -259,35 +259,32 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v0
+; GFX942-SDAG-NEXT: v_perm_b32 v3, v1, v1, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v8, v3, v3 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v8, v3, v3 offset1:1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v9, 16, v9
+; GFX942-SDAG-NEXT: v_add_u32_e32 v8, 16, v8
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB2_2
; GFX942-SDAG-NEXT: .LBB2_3: ; %Flow7
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
@@ -298,7 +295,7 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT: ds_write_b8 v0, v1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
@@ -1107,79 +1104,48 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v0
; GFX942-SDAG-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset0:32 offset1:33
-; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_add_u32_e32 v5, 0x100, v5
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB8_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -1268,77 +1234,46 @@ define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v1, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x800
; GFX942-SDAG-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:32 offset1:33
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB9_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 7a60a79f90a90..f9d7e5c27fd8a 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -15,6 +15,7 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX9_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX9_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -23,15 +24,16 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v4, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
; GFX9_W64-NEXT: ; implicit-def: $vgpr4
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX9_W64-NEXT: ; %bb.2:
; GFX9_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX9_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX9_W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1010_W32-LABEL: mubuf_vgpr:
@@ -41,22 +43,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v4, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
+; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v1
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
; GFX1010_W64-LABEL: mubuf_vgpr:
@@ -66,22 +70,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100_W32-LABEL: mubuf_vgpr:
@@ -92,21 +98,23 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v4, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
+; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v1
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100_W64-LABEL: mubuf_vgpr:
@@ -117,21 +125,23 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
+; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v1
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
; W64-O0-LABEL: mubuf_vgpr:
@@ -255,6 +265,7 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX9_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX9_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -263,8 +274,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX9_W64-NEXT: ; %bb.2:
@@ -281,14 +293,15 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
; GFX9_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9_W64-NEXT: ; implicit-def: $vgpr4
; GFX9_W64-NEXT: ; implicit-def: $vgpr8
+; GFX9_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX9_W64-NEXT: ; %bb.4:
; GFX9_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9_W64-NEXT: global_store_dword v[9:10], v13, off
+; GFX9_W64-NEXT: global_store_dword v[9:10], v1, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: global_store_dword v[11:12], v0, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
@@ -301,15 +314,17 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -326,14 +341,15 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
-; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
+; GFX1010_W32-NEXT: global_store_dword v[9:10], v1, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: global_store_dword v[11:12], v0, off
@@ -347,15 +363,17 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -372,14 +390,15 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
+; GFX1010_W64-NEXT: global_store_dword v[9:10], v1, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: global_store_dword v[11:12], v0, off
@@ -394,15 +413,17 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -419,13 +440,14 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
-; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[9:10], v1, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: global_store_b32 v[11:12], v0, off dlc
@@ -440,15 +462,17 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -465,13 +489,14 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
-; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[9:10], v1, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: global_store_b32 v[11:12], v0, off dlc
@@ -727,6 +752,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX9_W64-NEXT: s_mov_b64 s[12:13], exec
; GFX9_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX9_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX9_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX9_W64-NEXT: v_readfirstlane_b32 s11, v3
@@ -735,9 +761,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX9_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr0
; GFX9_W64-NEXT: ; implicit-def: $vgpr8
+; GFX9_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX9_W64-NEXT: ; %bb.2:
@@ -759,9 +786,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX9_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX9_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX9_W64-NEXT: s_nop 0
-; GFX9_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
-; GFX9_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9_W64-NEXT: buffer_load_format_x v1, v0, s[8:11], 0 idxen
+; GFX9_W64-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX9_W64-NEXT: ; implicit-def: $vgpr0
+; GFX9_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX9_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX9_W64-NEXT: ; %bb.5:
@@ -769,7 +797,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX9_W64-NEXT: .LBB2_6: ; %bb2
; GFX9_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX9_W64-NEXT: global_store_dword v[11:12], v1, off
; GFX9_W64-NEXT: s_waitcnt vmcnt(0)
; GFX9_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -779,21 +807,23 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -814,10 +844,11 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v1, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -825,7 +856,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v1, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -835,21 +866,23 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v8, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -870,10 +903,11 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v1, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -881,7 +915,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v1, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -891,22 +925,24 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -928,10 +964,11 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v1, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -939,7 +976,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v1, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -949,22 +986,24 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -986,10 +1025,11 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v1, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -997,7 +1037,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v1, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
index f08b68f9b8c38..2d3b18eab7b5e 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
@@ -10,7 +10,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_or_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1632 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
@@ -75,170 +75,161 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_writelane_b32 v63, s55, 13
; CHECK-NEXT: v_writelane_b32 v63, s30, 14
; CHECK-NEXT: v_writelane_b32 v63, s31, 15
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v30, v29
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:368
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:364
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:360
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:356
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:352
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:348
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:344
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:340
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:336
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:332
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:328
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:324
-; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:320
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:316
-; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:312
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:308
-; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:304
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:300
-; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:296
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:292
-; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:288
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:284
-; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:280
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:276
-; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:272
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:268
+; CHECK-NEXT: v_accvgpr_write_b32 a12, v30
+; CHECK-NEXT: v_accvgpr_write_b32 a0, v29
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v27
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v25
+; CHECK-NEXT: v_accvgpr_write_b32 a6, v23
+; CHECK-NEXT: v_accvgpr_write_b32 a8, v21
+; CHECK-NEXT: v_accvgpr_write_b32 a10, v20
+; CHECK-NEXT: v_mov_b32_e32 v20, v19
+; CHECK-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:136
+; CHECK-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:132
+; CHECK-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:128
+; CHECK-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:124
+; CHECK-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:120
+; CHECK-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:116
+; CHECK-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:112
+; CHECK-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:108
+; CHECK-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:104
+; CHECK-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:100
+; CHECK-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:96
+; CHECK-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:92
+; CHECK-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:88
+; CHECK-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:84
+; CHECK-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
+; CHECK-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:76
; CHECK-NEXT: v_and_b32_e32 v17, 1, v17
+; CHECK-NEXT: v_accvgpr_write_b32 a14, v24
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v17
+; CHECK-NEXT: v_and_b32_e32 v9, 1, v9
+; CHECK-NEXT: v_and_b32_e32 v8, 1, v8
+; CHECK-NEXT: v_and_b32_e32 v6, 1, v6
+; CHECK-NEXT: v_and_b32_e32 v2, 1, v2
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[78:79], 1, v9
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[88:89], 1, v8
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[92:93], 1, v6
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v2
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:784
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:780
+; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:392
+; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:388
+; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:384
+; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:380
+; CHECK-NEXT: buffer_load_ubyte v2, off, s[0:3], s32 offset:792
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:788
+; CHECK-NEXT: v_and_b32_e32 v3, 1, v3
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v3
+; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:376
+; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:372
+; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:368
+; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:364
+; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:360
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:356
+; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:352
+; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:348
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:344
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:340
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:336
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:332
+; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:328
+; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:324
+; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:320
+; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:316
+; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:312
+; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:308
+; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:304
+; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:300
+; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:296
+; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:292
+; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:288
+; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:284
+; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:280
+; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:276
+; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:272
+; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:268
+; CHECK-NEXT: v_and_b32_e32 v12, 1, v12
+; CHECK-NEXT: v_and_b32_e32 v11, 1, v11
+; CHECK-NEXT: v_and_b32_e32 v10, 1, v10
+; CHECK-NEXT: v_and_b32_e32 v7, 1, v7
+; CHECK-NEXT: v_and_b32_e32 v5, 1, v5
+; CHECK-NEXT: v_and_b32_e32 v4, 1, v4
+; CHECK-NEXT: v_accvgpr_write_b32 a48, v28
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[72:73], 1, v12
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[74:75], 1, v11
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[76:77], 1, v10
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[90:91], 1, v7
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[94:95], 1, v5
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v4
; CHECK-NEXT: v_and_b32_e32 v16, 1, v16
+; CHECK-NEXT: v_and_b32_e32 v15, 1, v15
+; CHECK-NEXT: v_and_b32_e32 v14, 1, v14
+; CHECK-NEXT: v_and_b32_e32 v13, 1, v13
; CHECK-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v16
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v15
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v14
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v13
+; CHECK-NEXT: v_accvgpr_write_b32 a50, v26
; CHECK-NEXT: s_mov_b64 s[48:49], 0
; CHECK-NEXT: s_mov_b64 s[52:53], 0
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:1496 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:1500 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v30, v27
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v25
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v23
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v21
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v20, v19
-; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:136
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: buffer_store_dword a16, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:132
-; CHECK-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:128
-; CHECK-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:124
-; CHECK-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:120
-; CHECK-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:116
-; CHECK-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:112
-; CHECK-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:108
-; CHECK-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:104
-; CHECK-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:100
-; CHECK-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:96
-; CHECK-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:92
-; CHECK-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:88
-; CHECK-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
-; CHECK-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:80
-; CHECK-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:76
-; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v15
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v14
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v13
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v12
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[72:73], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v11
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[74:75], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v10
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[76:77], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v9
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[78:79], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v8
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[88:89], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v7
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[90:91], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v6
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[92:93], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v5
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[94:95], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v4
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v3
-; CHECK-NEXT: s_waitcnt vmcnt(14)
-; CHECK-NEXT: v_and_b32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v2
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v1
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v0
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:392
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:388
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:384
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:380
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:376
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:372
-; CHECK-NEXT: buffer_load_ubyte v1, off, s[0:3], s32 offset:792
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788
-; CHECK-NEXT: s_waitcnt vmcnt(21)
-; CHECK-NEXT: v_and_b32_e32 v17, 1, v18
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_and_b32_e32 v18, 1, v19
-; CHECK-NEXT: s_waitcnt vmcnt(19)
-; CHECK-NEXT: v_and_b32_e32 v19, 1, v20
-; CHECK-NEXT: s_waitcnt vmcnt(18)
-; CHECK-NEXT: v_and_b32_e32 v20, 1, v21
-; CHECK-NEXT: s_waitcnt vmcnt(17)
+; CHECK-NEXT: buffer_store_dword a17, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a18, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a19, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a20, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a21, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a22, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a23, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a24, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a25, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a26, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a27, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a28, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a40, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a47, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; CHECK-NEXT: v_and_b32_e32 v17, 1, v19
+; CHECK-NEXT: v_and_b32_e32 v19, 1, v21
; CHECK-NEXT: v_and_b32_e32 v21, 1, v23
-; CHECK-NEXT: s_waitcnt vmcnt(16)
-; CHECK-NEXT: v_and_b32_e32 v23, 1, v32
-; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_and_b32_e32 v32, 1, v25
-; CHECK-NEXT: s_waitcnt vmcnt(14)
-; CHECK-NEXT: v_and_b32_e32 v25, 1, v33
-; CHECK-NEXT: s_waitcnt vmcnt(13)
-; CHECK-NEXT: v_and_b32_e32 v33, 1, v27
-; CHECK-NEXT: s_waitcnt vmcnt(12)
-; CHECK-NEXT: v_and_b32_e32 v27, 1, v34
-; CHECK-NEXT: s_waitcnt vmcnt(11)
-; CHECK-NEXT: v_and_b32_e32 v34, 1, v26
-; CHECK-NEXT: s_waitcnt vmcnt(10)
-; CHECK-NEXT: v_and_b32_e32 v26, 1, v29
-; CHECK-NEXT: s_waitcnt vmcnt(9)
-; CHECK-NEXT: v_and_b32_e32 v29, 1, v31
-; CHECK-NEXT: s_waitcnt vmcnt(8)
-; CHECK-NEXT: v_and_b32_e32 v31, 1, v35
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v32
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v33
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v34
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v31
+; CHECK-NEXT: v_and_b32_e32 v23, 1, v36
+; CHECK-NEXT: v_and_b32_e32 v36, 1, v25
+; CHECK-NEXT: v_and_b32_e32 v25, 1, v37
+; CHECK-NEXT: v_and_b32_e32 v37, 1, v27
+; CHECK-NEXT: v_and_b32_e32 v24, 1, v38
+; CHECK-NEXT: v_and_b32_e32 v38, 1, v29
+; CHECK-NEXT: v_and_b32_e32 v27, 1, v30
+; CHECK-NEXT: v_and_b32_e32 v30, 1, v31
+; CHECK-NEXT: v_and_b32_e32 v31, 1, v32
+; CHECK-NEXT: v_and_b32_e32 v32, 1, v33
+; CHECK-NEXT: v_and_b32_e32 v33, 1, v34
+; CHECK-NEXT: v_and_b32_e32 v34, 1, v35
+; CHECK-NEXT: v_and_b32_e32 v29, 1, v39
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v36
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v37
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v38
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v30
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v31
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v32
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v33
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v34
; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:264
; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:260
; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:256
@@ -271,849 +262,620 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148
; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144
; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:140
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v18
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v20
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v17
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v19
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v21
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v23
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v25
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v27
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v26
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v29
-; CHECK-NEXT: buffer_store_dword a18, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword a19, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a20, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a21, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a22, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a23, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a24, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a25, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a26, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a27, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a28, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a40, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
-; CHECK-NEXT: s_waitcnt vmcnt(60)
-; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a47, off, s[0:3], s32 offset:1252 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a48, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a49, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:712
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:708
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:704
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:700
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:696
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:692
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:688
-; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:684
-; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:680
-; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:676
-; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:672
-; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:668
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:664
-; CHECK-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:660
-; CHECK-NEXT: ; kill: def $agpr1 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr3 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr5 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr7 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr9 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr11 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr13 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr15 killed $sgpr4 killed $exec
+; CHECK-NEXT: v_and_b32_e32 v2, 1, v2
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v2
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v19
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v21
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v23
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v25
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v27
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v29
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v17
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v24
+; CHECK-NEXT: v_cmp_lt_i32_e64 s[38:39], 2, v6
; CHECK-NEXT: ; kill: def $agpr17 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr19 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr21 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr57 killed $sgpr4 killed $exec
; CHECK-NEXT: ; kill: def $agpr23 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr25 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr27 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr29 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr61 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr21 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr63 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr19 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $agpr1 killed $sgpr4 killed $exec
; CHECK-NEXT: ; kill: def $agpr31 killed $sgpr4 killed $exec
-; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:656
-; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:652
-; CHECK-NEXT: s_waitcnt vmcnt(48)
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
-; CHECK-NEXT: v_and_b32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v1
-; CHECK-NEXT: s_waitcnt vmcnt(32)
-; CHECK-NEXT: buffer_store_dword a0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a10, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a11, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a12, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a13, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a14, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a15, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a16, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a17, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a18, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a19, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a20, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a21, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a22, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a23, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a24, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a25, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a26, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a27, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a28, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:412
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:408
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:416
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:404
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:420
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:400
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:396
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:424
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:440
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:436
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:444
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:432
+; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:1252 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:712
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:448
+; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:708
+; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:704
+; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:700
+; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:696
+; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:692
+; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:688
+; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:684
+; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:680
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:676
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:672
+; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:668
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:664
+; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:660
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:656
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:412
+; CHECK-NEXT: ; kill: def $vgpr31 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr33 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr35 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr37 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr39 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr41 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr43 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr45 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr47 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr49 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr51 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr53 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr55 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr57 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr59 killed $sgpr4 killed $exec
+; CHECK-NEXT: ; kill: def $vgpr61 killed $sgpr4 killed $exec
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:428
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:408
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:452
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:416
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:784
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:404
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:780
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:420
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:8
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:4
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:400
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:424
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:456
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:584
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:580
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:576
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:572
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:568
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:564
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:560
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:556
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:552
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:548
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:544
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:540
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:536
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:532
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:528
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:440
+; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:436
+; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:444
+; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:432
+; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:448
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:428
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:452
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:584
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:580
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:576
+; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:572
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Spill
-; CHECK-NEXT: s_waitcnt vmcnt(2)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:1564 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:1568 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:1572 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:1576 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:1580 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1584 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1588 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:1592 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:1596 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:1600 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:1604 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1608 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1612 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:1616 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:1620 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1624 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Spill
-; CHECK-NEXT: v_cmp_lt_i32_e64 s[38:39], 2, v0
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:568
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:564
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:560
+; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:556
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552
+; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:548
+; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:544
+; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:540
+; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:536
+; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:532
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:528
+; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:524
+; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:8
+; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:4
+; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:652
+; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:396
+; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:456
; CHECK-NEXT: s_and_saveexec_b64 s[50:51], s[38:39]
; CHECK-NEXT: s_xor_b64 s[50:51], exec, s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_4
; CHECK-NEXT: ; %bb.1: ; %LeafBlock46
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[38:39], 3, v0
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[38:39], 3, v6
; CHECK-NEXT: s_mov_b64 s[54:55], -1
; CHECK-NEXT: s_and_saveexec_b64 s[52:53], s[38:39]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.2: ; %bb2
-; CHECK-NEXT: v_accvgpr_write_b32 a10, v42
-; CHECK-NEXT: v_accvgpr_write_b32 a14, v48
-; CHECK-NEXT: v_accvgpr_write_b32 a4, v56
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v50
-; CHECK-NEXT: v_accvgpr_write_b32 a11, v43
-; CHECK-NEXT: v_accvgpr_write_b32 a8, v40
-; CHECK-NEXT: v_accvgpr_write_b32 a15, v49
-; CHECK-NEXT: v_accvgpr_write_b32 a5, v57
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v51
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_write_b32 a16, v58
-; CHECK-NEXT: v_accvgpr_write_b32 a17, v59
-; CHECK-NEXT: v_accvgpr_read_b32 v60, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a40
-; CHECK-NEXT: s_xor_b64 s[54:55], exec, -1
-; CHECK-NEXT: s_waitcnt vmcnt(29)
-; CHECK-NEXT: v_xor_b32_e32 v28, v28, v18
-; CHECK-NEXT: s_waitcnt vmcnt(28)
-; CHECK-NEXT: v_xor_b32_e32 v0, v29, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a9, v0
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a27
-; CHECK-NEXT: s_waitcnt vmcnt(26)
-; CHECK-NEXT: v_xor_b32_e32 v29, v31, v29
-; CHECK-NEXT: s_waitcnt vmcnt(24)
-; CHECK-NEXT: v_xor_b32_e32 v0, v33, v7
-; CHECK-NEXT: v_accvgpr_write_b32 a27, v0
-; CHECK-NEXT: v_xor_b32_e32 v0, v30, v4
-; CHECK-NEXT: v_accvgpr_write_b32 a12, v0
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_xor_b32_e32 v62, v37, v11
-; CHECK-NEXT: v_xor_b32_e32 v0, v32, v6
-; CHECK-NEXT: v_xor_b32_e32 v32, v34, v8
-; CHECK-NEXT: v_xor_b32_e32 v34, v36, v10
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a46
-; CHECK-NEXT: v_xor_b32_e32 v33, v35, v9
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_xor_b32_e32 v35, v39, v13
-; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_xor_b32_e32 v10, v44, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a44
-; CHECK-NEXT: s_waitcnt vmcnt(11)
-; CHECK-NEXT: v_xor_b32_e32 v6, v48, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a42
-; CHECK-NEXT: v_xor_b32_e32 v9, v47, v21
; CHECK-NEXT: s_waitcnt vmcnt(8)
-; CHECK-NEXT: v_xor_b32_e32 v21, v51, v25
-; CHECK-NEXT: v_accvgpr_write_b32 a36, v0
-; CHECK-NEXT: s_waitcnt vmcnt(6)
-; CHECK-NEXT: v_xor_b32_e32 v13, v53, v31
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a29
-; CHECK-NEXT: s_waitcnt vmcnt(4)
-; CHECK-NEXT: v_xor_b32_e32 v1, v55, v31
-; CHECK-NEXT: s_waitcnt vmcnt(3)
-; CHECK-NEXT: v_xor_b32_e32 v25, v56, v60
-; CHECK-NEXT: v_xor_b32_e32 v8, v46, v20
-; CHECK-NEXT: v_xor_b32_e32 v20, v50, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a38
-; CHECK-NEXT: v_xor_b32_e32 v0, v54, v58
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
-; CHECK-NEXT: v_xor_b32_e32 v11, v45, v19
-; CHECK-NEXT: v_xor_b32_e32 v19, v27, v5
-; CHECK-NEXT: v_xor_b32_e32 v4, v52, v30
-; CHECK-NEXT: v_xor_b32_e32 v30, v26, v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v5, 17, v25
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 14, v[0:1]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v5
-; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v13
-; CHECK-NEXT: v_xor_b32_e32 v18, v38, v12
-; CHECK-NEXT: v_lshlrev_b32_e32 v12, 18, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 13, v[4:5]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v12
-; CHECK-NEXT: v_xor_b32_e32 v7, v49, v23
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a37
+; CHECK-NEXT: v_xor_b32_e32 v6, v59, v28
; CHECK-NEXT: s_waitcnt vmcnt(2)
-; CHECK-NEXT: v_xor_b32_e32 v23, v57, v31
-; CHECK-NEXT: v_xor_b32_e32 v17, v43, v17
-; CHECK-NEXT: v_xor_b32_e32 v16, v42, v16
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_xor_b32_e32 v15, v41, v15
-; CHECK-NEXT: v_xor_b32_e32 v14, v40, v14
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_accvgpr_read_b32 v57, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v43, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v49, a15
-; CHECK-NEXT: v_accvgpr_read_b32 v51, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v59, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v56, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v42, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v40, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v48, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v50, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a16
-; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: v_add_co_u32_e64 v12, s[38:39], 24, v36
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_addc_co_u32_e64 v13, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:80
+; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
+; CHECK-NEXT: v_xor_b32_e32 v14, v58, v11
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v6
+; CHECK-NEXT: v_xor_b32_e32 v26, v60, v10
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v26
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 14, v[14:15]
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v6
+; CHECK-NEXT: v_xor_b32_e32 v6, v57, v26
+; CHECK-NEXT: v_xor_b32_e32 v16, v56, v7
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v14
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 13, v[16:17]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v6
+; CHECK-NEXT: v_add_co_u32_e64 v6, s[38:39], 24, v8
+; CHECK-NEXT: v_addc_co_u32_e64 v7, s[38:39], 0, v9, s[38:39]
+; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] offset:80
+; CHECK-NEXT: v_lshlrev_b32_e32 v14, 19, v16
+; CHECK-NEXT: v_xor_b32_e32 v11, v55, v24
+; CHECK-NEXT: v_xor_b32_e32 v10, v54, v21
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 12, v[10:11]
+; CHECK-NEXT: v_xor_b32_e32 v11, v53, v22
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v14
+; CHECK-NEXT: v_xor_b32_e32 v14, v52, v3
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v11
+; CHECK-NEXT: v_lshlrev_b32_e32 v16, 20, v10
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 11, v[14:15]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v16
+; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v14, 21, v14
+; CHECK-NEXT: v_xor_b32_e32 v11, v51, v20
+; CHECK-NEXT: v_xor_b32_e32 v10, v50, v5
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 10, v[10:11]
+; CHECK-NEXT: v_xor_b32_e32 v11, v49, v18
+; CHECK-NEXT: v_xor_b32_e32 v4, v48, v4
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v11
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v14
+; CHECK-NEXT: v_lshlrev_b32_e32 v14, 22, v10
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 9, v[4:5]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v14
+; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] offset:48
+; CHECK-NEXT: v_xor_b32_e32 v14, v44, v2
+; CHECK-NEXT: v_xor_b32_e32 v11, v43, v12
+; CHECK-NEXT: v_xor_b32_e32 v10, v42, v25
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 6, v[10:11]
+; CHECK-NEXT: v_xor_b32_e32 v11, v41, v10
+; CHECK-NEXT: v_lshlrev_b32_e32 v15, 25, v14
+; CHECK-NEXT: v_xor_b32_e32 v24, v40, v23
+; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v11
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v15
+; CHECK-NEXT: v_lshlrev_b32_e32 v15, 26, v10
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 5, v[24:25]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v15
+; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] offset:16
+; CHECK-NEXT: v_xor_b32_e32 v16, v38, v62
+; CHECK-NEXT: v_xor_b32_e32 v10, v45, v14
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v10
+; CHECK-NEXT: v_xor_b32_e32 v12, v46, v19
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 7, v[14:15]
+; CHECK-NEXT: v_lshlrev_b32_e32 v13, 24, v12
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v13
+; CHECK-NEXT: flat_store_dwordx2 v[6:7], v[10:11] offset:32
+; CHECK-NEXT: v_xor_b32_e32 v10, v37, v6
+; CHECK-NEXT: v_xor_b32_e32 v14, v36, v29
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v10
+; CHECK-NEXT: v_lshlrev_b32_e32 v13, 28, v16
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 3, v[14:15]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v13
; CHECK-NEXT: s_movk_i32 s38, 0x7c
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 19, v4
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 12, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 20, v20
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 11, v[6:7]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 22, v8
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 21, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 10, v[8:9]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 9, v[10:11]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 26, v18
-; CHECK-NEXT: v_add_co_u32_e64 v0, s[38:39], s38, v36
-; CHECK-NEXT: v_addc_co_u32_e64 v1, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v23
+; CHECK-NEXT: flat_store_dwordx2 v[6:7], v[10:11]
+; CHECK-NEXT: v_add_co_u32_e64 v6, s[38:39], s38, v8
+; CHECK-NEXT: v_addc_co_u32_e64 v7, s[38:39], 0, v9, s[38:39]
+; CHECK-NEXT: v_xor_b32_e32 v10, v61, v30
+; CHECK-NEXT: v_lshrrev_b32_e32 v11, 15, v10
; CHECK-NEXT: s_movk_i32 s38, 0x78
-; CHECK-NEXT: flat_store_short v[0:1], v2
-; CHECK-NEXT: v_add_co_u32_e64 v0, s[38:39], s38, v36
-; CHECK-NEXT: v_alignbit_b32 v2, v23, v25, 15
-; CHECK-NEXT: v_addc_co_u32_e64 v1, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_and_b32_e32 v0, 0x7fffffff, v19
-; CHECK-NEXT: v_lshlrev_b32_e32 v1, 31, v28
-; CHECK-NEXT: v_or_b32_e32 v31, v0, v1
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v10
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 8, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 24, v16
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 7, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v35
-; CHECK-NEXT: flat_store_dwordx2 v[12:13], v[0:1] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 25, v14
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 6, v[18:19]
-; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v62
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 5, v[34:35]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a36
-; CHECK-NEXT: v_mov_b32_e32 v0, v33
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 4, v[32:33]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 27, v34
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:32
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a36
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 3, v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 28, v32
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[12:13], v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v0, v29
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a12
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v6
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a12
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[32:33], 1, v[28:29]
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v4
-; CHECK-NEXT: v_or_b32_e32 v33, v33, v0
-; CHECK-NEXT: flat_store_dwordx4 v[36:37], v[30:33]
+; CHECK-NEXT: flat_store_short v[6:7], v11
+; CHECK-NEXT: v_add_co_u32_e64 v6, s[38:39], s38, v8
+; CHECK-NEXT: v_alignbit_b32 v10, v10, v26, 15
+; CHECK-NEXT: v_addc_co_u32_e64 v7, s[38:39], 0, v9, s[38:39]
+; CHECK-NEXT: flat_store_dword v[6:7], v10
+; CHECK-NEXT: v_xor_b32_e32 v7, v33, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v15, 23, v4
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
+; CHECK-NEXT: v_xor_b32_e32 v6, v47, v16
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v6
+; CHECK-NEXT: v_xor_b32_e32 v10, v31, v0
+; CHECK-NEXT: v_xor_b32_e32 v6, v32, v27
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v10
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 8, v[12:13]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v15
+; CHECK-NEXT: v_lshlrev_b32_e32 v12, 31, v6
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[10:11] offset:64
+; CHECK-NEXT: v_or_b32_e32 v11, v17, v12
+; CHECK-NEXT: v_xor_b32_e32 v12, v39, v8
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v12
+; CHECK-NEXT: v_lshlrev_b32_e32 v19, 27, v24
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 4, v[16:17]
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v19
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[12:13] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_lshlrev_b32_e32 v5, 29, v14
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
+; CHECK-NEXT: s_xor_b64 s[54:55], exec, -1
+; CHECK-NEXT: v_xor_b32_e32 v10, v2, v30
+; CHECK-NEXT: v_xor_b32_e32 v2, v34, v28
+; CHECK-NEXT: v_xor_b32_e32 v15, v35, v4
+; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v15
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 2, v[2:3]
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v5
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[12:13] offset:16
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 1, v[6:7]
+; CHECK-NEXT: v_lshlrev_b32_e32 v5, 30, v2
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v5
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[10:13]
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
; CHECK-NEXT: .LBB0_3: ; %Flow52
; CHECK-NEXT: s_or_b64 exec, exec, s[52:53]
; CHECK-NEXT: s_and_b64 s[52:53], s[54:55], exec
-; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr6
; CHECK-NEXT: .LBB0_4: ; %Flow51
; CHECK-NEXT: s_andn2_saveexec_b64 s[50:51], s[50:51]
; CHECK-NEXT: ; %bb.5: ; %LeafBlock
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[38:39], 2, v0
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[38:39], 2, v6
; CHECK-NEXT: s_andn2_b64 s[48:49], s[52:53], exec
; CHECK-NEXT: s_and_b64 s[38:39], s[38:39], exec
; CHECK-NEXT: s_or_b64 s[52:53], s[48:49], s[38:39]
; CHECK-NEXT: s_mov_b64 s[48:49], exec
; CHECK-NEXT: ; %bb.6: ; %Flow53
; CHECK-NEXT: s_or_b64 exec, exec, s[50:51]
-; CHECK-NEXT: s_mov_b64 s[38:39], exec
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
-; CHECK-NEXT: s_and_b64 s[50:51], s[38:39], s[52:53]
-; CHECK-NEXT: s_mov_b64 exec, s[50:51]
+; CHECK-NEXT: s_and_saveexec_b64 s[38:39], s[52:53]
; CHECK-NEXT: s_cbranch_execz .LBB0_44
; CHECK-NEXT: ; %bb.7: ; %bb0
; CHECK-NEXT: s_and_saveexec_b64 s[50:51], s[46:47]
; CHECK-NEXT: s_xor_b64 s[50:51], exec, s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_41
; CHECK-NEXT: ; %bb.8: ; %bb0b
-; CHECK-NEXT: s_mov_b64 s[52:53], exec
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
-; CHECK-NEXT: s_and_b64 s[36:37], s[52:53], s[36:37]
-; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr9
-; CHECK-NEXT: s_mov_b64 exec, s[36:37]
+; CHECK-NEXT: s_and_saveexec_b64 s[52:53], s[36:37]
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.9: ; %cond.load
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[8:9], v[0:1]
+; CHECK-NEXT: flat_load_dwordx2 v[18:19], v[0:1]
; CHECK-NEXT: .LBB0_10: ; %else
; CHECK-NEXT: s_or_b64 exec, exec, s[52:53]
; CHECK-NEXT: s_mov_b64 s[36:37], exec
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1496 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1500 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v15, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a6
+; CHECK-NEXT: v_accvgpr_read_b32 v25, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v17, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a50
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a48
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v37, a25
+; CHECK-NEXT: v_accvgpr_read_b32 v10, a26
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a28
; CHECK-NEXT: s_and_b64 s[34:35], s[36:37], s[34:35]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a10
+; CHECK-NEXT: v_accvgpr_read_b32 v14, a8
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a51
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a49
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a12
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a24
+; CHECK-NEXT: v_accvgpr_read_b32 v11, a27
+; CHECK-NEXT: v_accvgpr_read_b32 v7, a29
; CHECK-NEXT: s_mov_b64 exec, s[34:35]
; CHECK-NEXT: s_cbranch_execz .LBB0_12
; CHECK-NEXT: ; %bb.11: ; %cond.load1
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[0:1] offset:8
+; CHECK-NEXT: flat_load_dwordx2 v[20:21], v[0:1] offset:8
; CHECK-NEXT: .LBB0_12: ; %else2
; CHECK-NEXT: s_or_b64 exec, exec, s[36:37]
; CHECK-NEXT: s_and_saveexec_b64 s[34:35], s[30:31]
; CHECK-NEXT: s_cbranch_execz .LBB0_14
; CHECK-NEXT: ; %bb.13: ; %cond.load4
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[12:13], v[0:1] offset:16
; CHECK-NEXT: .LBB0_14: ; %else5
; CHECK-NEXT: s_or_b64 exec, exec, s[34:35]
; CHECK-NEXT: s_and_saveexec_b64 s[30:31], s[94:95]
; CHECK-NEXT: s_cbranch_execz .LBB0_16
; CHECK-NEXT: ; %bb.15: ; %cond.load7
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[14:15], v[0:1] offset:24
; CHECK-NEXT: .LBB0_16: ; %else8
; CHECK-NEXT: s_or_b64 exec, exec, s[30:31]
; CHECK-NEXT: s_and_saveexec_b64 s[94:95], s[92:93]
; CHECK-NEXT: s_cbranch_execz .LBB0_18
; CHECK-NEXT: ; %bb.17: ; %cond.load10
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[22:23], v[0:1] offset:32
; CHECK-NEXT: .LBB0_18: ; %else11
; CHECK-NEXT: s_or_b64 exec, exec, s[94:95]
; CHECK-NEXT: s_and_saveexec_b64 s[92:93], s[90:91]
; CHECK-NEXT: s_cbranch_execz .LBB0_20
; CHECK-NEXT: ; %bb.19: ; %cond.load13
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[16:17], v[0:1] offset:40
+; CHECK-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:40
; CHECK-NEXT: .LBB0_20: ; %else14
; CHECK-NEXT: s_or_b64 exec, exec, s[92:93]
; CHECK-NEXT: s_and_saveexec_b64 s[90:91], s[88:89]
; CHECK-NEXT: s_cbranch_execz .LBB0_22
; CHECK-NEXT: ; %bb.21: ; %cond.load16
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[0:1] offset:48
; CHECK-NEXT: .LBB0_22: ; %else17
; CHECK-NEXT: s_or_b64 exec, exec, s[90:91]
; CHECK-NEXT: s_and_saveexec_b64 s[88:89], s[78:79]
; CHECK-NEXT: s_cbranch_execz .LBB0_24
; CHECK-NEXT: ; %bb.23: ; %cond.load19
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[18:19], v[0:1] offset:56
+; CHECK-NEXT: flat_load_dwordx2 v[16:17], v[0:1] offset:56
; CHECK-NEXT: .LBB0_24: ; %else20
; CHECK-NEXT: s_or_b64 exec, exec, s[88:89]
; CHECK-NEXT: s_and_saveexec_b64 s[78:79], s[76:77]
; CHECK-NEXT: s_cbranch_execz .LBB0_26
; CHECK-NEXT: ; %bb.25: ; %cond.load22
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[26:27], v[0:1] offset:64
; CHECK-NEXT: .LBB0_26: ; %else23
; CHECK-NEXT: s_or_b64 exec, exec, s[78:79]
; CHECK-NEXT: s_and_saveexec_b64 s[76:77], s[74:75]
; CHECK-NEXT: s_cbranch_execz .LBB0_28
; CHECK-NEXT: ; %bb.27: ; %cond.load25
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[20:21], v[0:1] offset:72
+; CHECK-NEXT: flat_load_dwordx2 v[34:35], v[0:1] offset:72
; CHECK-NEXT: .LBB0_28: ; %else26
; CHECK-NEXT: s_or_b64 exec, exec, s[76:77]
; CHECK-NEXT: s_and_saveexec_b64 s[74:75], s[72:73]
; CHECK-NEXT: s_cbranch_execz .LBB0_30
; CHECK-NEXT: ; %bb.29: ; %cond.load28
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[28:29], v[0:1] offset:80
; CHECK-NEXT: .LBB0_30: ; %else29
; CHECK-NEXT: s_or_b64 exec, exec, s[74:75]
; CHECK-NEXT: s_and_saveexec_b64 s[72:73], s[62:63]
; CHECK-NEXT: s_cbranch_execz .LBB0_32
; CHECK-NEXT: ; %bb.31: ; %cond.load31
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[32:33], v[0:1] offset:88
; CHECK-NEXT: .LBB0_32: ; %else32
; CHECK-NEXT: s_or_b64 exec, exec, s[72:73]
; CHECK-NEXT: s_and_saveexec_b64 s[62:63], s[60:61]
; CHECK-NEXT: s_cbranch_execz .LBB0_34
; CHECK-NEXT: ; %bb.33: ; %cond.load34
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[30:31], v[0:1] offset:96
; CHECK-NEXT: .LBB0_34: ; %else35
; CHECK-NEXT: s_or_b64 exec, exec, s[62:63]
; CHECK-NEXT: s_and_saveexec_b64 s[60:61], s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_36
; CHECK-NEXT: ; %bb.35: ; %cond.load37
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:104
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; CHECK-NEXT: flat_load_dwordx2 v[36:37], v[0:1] offset:104
; CHECK-NEXT: .LBB0_36: ; %else38
; CHECK-NEXT: s_or_b64 exec, exec, s[60:61]
; CHECK-NEXT: s_and_saveexec_b64 s[58:59], s[56:57]
; CHECK-NEXT: s_cbranch_execz .LBB0_38
; CHECK-NEXT: ; %bb.37: ; %cond.load40
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 a[26:27], v[0:1] offset:112
+; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[0:1] offset:112
; CHECK-NEXT: .LBB0_38: ; %else41
; CHECK-NEXT: s_or_b64 exec, exec, s[58:59]
; CHECK-NEXT: s_and_saveexec_b64 s[56:57], vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_40
; CHECK-NEXT: ; %bb.39: ; %cond.load43
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 a[28:29], v[0:1] offset:120
+; CHECK-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:120
; CHECK-NEXT: .LBB0_40: ; %else44
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v8
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a29
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v5
+; CHECK-NEXT: v_lshrrev_b32_e32 v4, 15, v7
+; CHECK-NEXT: flat_store_short v[0:1], v4
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v8
+; CHECK-NEXT: v_alignbit_b32 v4, v7, v6, 15
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: flat_store_dword v[0:1], v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v20
+; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v19
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_or_b32_e32 v19, v1, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 14, v[10:11]
+; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v0
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v8
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 18, v10
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 13, v[36:37]
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v10
; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:80
; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 19, v36
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 12, v[30:31]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 20, v30
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 11, v[32:33]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v10
; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:64
+; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 21, v32
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 10, v[28:29]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 22, v28
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 9, v[34:35]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v10
; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
-; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 23, v34
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 8, v[26:27]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[4:5] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 24, v26
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 7, v[16:17]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
+; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 25, v16
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 6, v[24:25]
+; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 26, v24
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 5, v[2:3]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v10
; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:16
; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 27, v2
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 4, v[22:23]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[4:5] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 28, v22
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 3, v[14:15]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v34
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
-; CHECK-NEXT: flat_store_short v[0:1], v2
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v34
-; CHECK-NEXT: v_alignbit_b32 v2, v5, v4, 15
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
-; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v3, a27
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v10
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v9
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
-; CHECK-NEXT: v_or_b32_e32 v9, v1, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v4
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
-; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v0
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v34
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[36:37]
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v30
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v36
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[30:31]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[32:33]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v32
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[28:29]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v24
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v20
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[26:27]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v26
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[18:19]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v18
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[24:25]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v16
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[22:23]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v22
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v14
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 29, v14
; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[12:13]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[0:1] offset:16
+; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
+; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[0:1] offset:16
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v12
-; CHECK-NEXT: v_or_b32_e32 v11, v11, v0
-; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[8:11]
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 1, v[20:21]
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v0
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[18:21]
+; CHECK-NEXT: ; implicit-def: $vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61
+; CHECK-NEXT: ; implicit-def: $vgpr12
; CHECK-NEXT: .LBB0_41: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[56:57], s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_43
; CHECK-NEXT: ; %bb.42: ; %bb0a
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v8
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
-; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
-; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
-; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
-; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v36
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v52
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v39
-; CHECK-NEXT: flat_store_short v[0:1], v2
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v52
-; CHECK-NEXT: v_alignbit_b32 v2, v39, v38, 15
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
-; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v10
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v9
+; CHECK-NEXT: v_lshrrev_b32_e32 v5, 15, v61
+; CHECK-NEXT: flat_store_short v[0:1], v5
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v8
+; CHECK-NEXT: v_alignbit_b32 v5, v61, v60, 15
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: flat_store_dword v[0:1], v5
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v32
+; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v31
+; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v59
+; CHECK-NEXT: v_or_b32_e32 v13, v1, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v60
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 14, v[58:59]
+; CHECK-NEXT: v_and_b32_e32 v57, 0x7fffffff, v57
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v0
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v8
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v58
+; CHECK-NEXT: v_lshrrev_b64 v[18:19], 13, v[56:57]
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: v_or_b32_e32 v19, v19, v6
+; CHECK-NEXT: v_and_b32_e32 v55, 0x7fffffff, v55
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:80
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 19, v56
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 12, v[54:55]
+; CHECK-NEXT: v_and_b32_e32 v53, 0x7fffffff, v53
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v54
+; CHECK-NEXT: v_lshrrev_b64 v[18:19], 11, v[52:53]
+; CHECK-NEXT: v_or_b32_e32 v19, v19, v6
+; CHECK-NEXT: v_and_b32_e32 v51, 0x7fffffff, v51
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 21, v52
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 10, v[50:51]
+; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v49
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v50
+; CHECK-NEXT: v_lshrrev_b64 v[18:19], 9, v[48:49]
+; CHECK-NEXT: v_and_b32_e32 v47, 0x7fffffff, v47
+; CHECK-NEXT: v_or_b32_e32 v19, v19, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 23, v48
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 8, v[46:47]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v10
+; CHECK-NEXT: v_and_b32_e32 v45, 0x7fffffff, v45
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[6:7] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 24, v46
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 7, v[44:45]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v10
+; CHECK-NEXT: v_and_b32_e32 v43, 0x7fffffff, v43
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:48
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[6:7] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 25, v44
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 6, v[42:43]
+; CHECK-NEXT: v_and_b32_e32 v41, 0x7fffffff, v41
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v42
+; CHECK-NEXT: v_lshrrev_b64 v[18:19], 5, v[40:41]
+; CHECK-NEXT: v_and_b32_e32 v39, 0x7fffffff, v39
+; CHECK-NEXT: v_or_b32_e32 v19, v19, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 27, v40
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 4, v[38:39]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v10
; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
-; CHECK-NEXT: v_or_b32_e32 v9, v1, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v38
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[36:37]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v0
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v52
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[34:35]
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v34
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[32:33]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[30:31]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v30
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[28:29]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[26:27]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v20
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v26
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[24:25]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v24
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[22:23]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v22
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[18:19]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v18
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v16
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v14
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[12:13]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[0:1] offset:16
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v12
-; CHECK-NEXT: v_or_b32_e32 v11, v11, v0
-; CHECK-NEXT: flat_store_dwordx4 v[52:53], v[8:11]
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[6:7] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 28, v38
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 3, v[36:37]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v10
+; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:16
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[6:7]
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 29, v36
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[34:35]
+; CHECK-NEXT: v_or_b32_e32 v1, v1, v6
+; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[0:1] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v34
+; CHECK-NEXT: v_lshrrev_b64 v[14:15], 1, v[32:33]
+; CHECK-NEXT: v_or_b32_e32 v15, v15, v0
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[12:15]
; CHECK-NEXT: .LBB0_43: ; %Flow48
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
; CHECK-NEXT: s_andn2_b64 s[48:49], s[48:49], exec
; CHECK-NEXT: .LBB0_44: ; %Flow54
; CHECK-NEXT: s_or_b64 exec, exec, s[38:39]
+; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
; CHECK-NEXT: s_and_saveexec_b64 s[56:57], s[48:49]
; CHECK-NEXT: s_cbranch_execz .LBB0_49
; CHECK-NEXT: ; %bb.45: ; %bb1
@@ -1121,226 +883,217 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_xor_b64 s[46:47], exec, s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_47
; CHECK-NEXT: ; %bb.46: ; %bb1b
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
-; CHECK-NEXT: v_lshrrev_b32_e32 v33, 15, v31
-; CHECK-NEXT: v_alignbit_b32 v36, v31, v32, 15
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1564 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1568 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1572 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1576 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1580 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1584 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1588 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1592 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1596 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1600 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1604 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1608 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1612 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1616 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1620 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1624 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v32
-; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v59
-; CHECK-NEXT: v_and_b32_e32 v51, 0x7fffffff, v51
-; CHECK-NEXT: v_and_b32_e32 v57, 0x7fffffff, v57
-; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v49
-; CHECK-NEXT: v_and_b32_e32 v43, 0x7fffffff, v43
-; CHECK-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_short v[0:1], v33
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_dword v[0:1], v36
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v42
+; CHECK-NEXT: v_lshrrev_b32_e32 v4, 15, v31
+; CHECK-NEXT: v_and_b32_e32 v6, 0x7fffffff, v1
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a62
+; CHECK-NEXT: v_alignbit_b32 v5, v31, v30, 15
+; CHECK-NEXT: v_accvgpr_read_b32 v10, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a19
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a63
; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
-; CHECK-NEXT: v_or_b32_e32 v41, v4, v0
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
+; CHECK-NEXT: v_lshlrev_b32_e32 v7, 17, v30
+; CHECK-NEXT: v_accvgpr_read_b32 v11, a17
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_lshrrev_b64 v[16:17], 1, v[10:11]
+; CHECK-NEXT: v_accvgpr_read_b32 v14, a58
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v8
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: flat_store_short v[0:1], v4
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v8
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; CHECK-NEXT: flat_store_dword v[0:1], v5
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v10
+; CHECK-NEXT: v_or_b32_e32 v15, v6, v0
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v8
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[8:9]
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[12:13]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v7
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a60
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshrrev_b64 v[42:43], 1, v[42:43]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v8
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a33
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_accvgpr_read_b32 v7, a61
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a56
; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a57
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[6:7]
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[8:9]
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a34
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a22
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a35
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v8
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a24
+; CHECK-NEXT: v_accvgpr_read_b32 v7, a23
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a20
; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a25
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a21
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[6:7]
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[8:9]
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
+; CHECK-NEXT: ; implicit-def: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+; CHECK-NEXT: ; kill: killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a22
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v8
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[58:59]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v58
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[50:51]
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a20
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v12
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[6:7]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v6
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a21
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[2:3] offset:64
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v50
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v6
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[6:7]
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[8:9]
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a6
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a7
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v8
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[56:57]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v12
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[6:7]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v6
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v56
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[2:3] offset:32
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[6:7]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v5
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v6
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[48:49]
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[4:5]
; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[0:1] offset:16
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v48
-; CHECK-NEXT: v_or_b32_e32 v43, v43, v0
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[40:43]
+; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[0:1] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v4
+; CHECK-NEXT: v_or_b32_e32 v17, v17, v0
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[14:17]
; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: .LBB0_47: ; %Flow49
; CHECK-NEXT: s_andn2_saveexec_b64 s[46:47], s[46:47]
; CHECK-NEXT: s_cbranch_execz .LBB0_49
; CHECK-NEXT: ; %bb.48: ; %bb1a
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:1304 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:1308 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:1312 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a11, off, s[0:3], s32 offset:1316 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:1320 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a13, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a15, off, s[0:3], s32 offset:1332 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:1336 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a11, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a13, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a15, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:1304 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:1308 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
; CHECK-NEXT: v_accvgpr_read_b32 v30, a32
+; CHECK-NEXT: v_accvgpr_read_b32 v38, a40
+; CHECK-NEXT: v_accvgpr_read_b32 v39, a41
+; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v28, a28
; CHECK-NEXT: v_accvgpr_read_b32 v31, a33
; CHECK-NEXT: v_accvgpr_read_b32 v32, a34
; CHECK-NEXT: v_accvgpr_read_b32 v33, a35
; CHECK-NEXT: v_accvgpr_read_b32 v34, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v36, a38
+; CHECK-NEXT: v_accvgpr_read_b32 v48, a50
; CHECK-NEXT: v_accvgpr_read_b32 v32, a28
; CHECK-NEXT: v_accvgpr_read_b32 v28, a60
; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
@@ -1353,18 +1106,18 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v26, a26
; CHECK-NEXT: v_accvgpr_read_b32 v27, a27
; CHECK-NEXT: v_accvgpr_read_b32 v29, a29
-; CHECK-NEXT: v_accvgpr_read_b32 v37, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v38, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v39, a41
+; CHECK-NEXT: v_accvgpr_read_b32 v49, a51
+; CHECK-NEXT: v_accvgpr_read_b32 v50, a52
+; CHECK-NEXT: v_accvgpr_read_b32 v51, a53
; CHECK-NEXT: v_accvgpr_read_b32 v59, a61
-; CHECK-NEXT: v_cndmask_b32_e64 v36, v28, v32, s[44:45]
+; CHECK-NEXT: v_cndmask_b32_e64 v48, v28, v32, s[44:45]
; CHECK-NEXT: v_accvgpr_read_b32 v33, a31
; CHECK-NEXT: v_accvgpr_read_b32 v31, a63
; CHECK-NEXT: v_accvgpr_read_b32 v34, a30
; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
-; CHECK-NEXT: v_cndmask_b32_e64 v37, v59, v29, s[44:45]
-; CHECK-NEXT: v_cndmask_b32_e64 v39, v31, v33, s[40:41]
-; CHECK-NEXT: v_cndmask_b32_e64 v38, v30, v34, s[40:41]
+; CHECK-NEXT: v_cndmask_b32_e64 v49, v59, v29, s[44:45]
+; CHECK-NEXT: v_cndmask_b32_e64 v51, v31, v33, s[40:41]
+; CHECK-NEXT: v_cndmask_b32_e64 v50, v30, v34, s[40:41]
; CHECK-NEXT: v_accvgpr_read_b32 v31, a27
; CHECK-NEXT: v_accvgpr_read_b32 v16, a16
; CHECK-NEXT: v_accvgpr_read_b32 v17, a17
@@ -1379,7 +1132,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v49, a51
+; CHECK-NEXT: v_accvgpr_read_b32 v53, a55
; CHECK-NEXT: v_accvgpr_read_b32 v29, a25
; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
@@ -1390,7 +1143,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v25, a57
; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
; CHECK-NEXT: v_accvgpr_read_b32 v27, a59
-; CHECK-NEXT: v_cndmask_b32_e64 v49, v25, v29, s[42:43]
+; CHECK-NEXT: v_cndmask_b32_e64 v53, v25, v29, s[42:43]
; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
@@ -1405,9 +1158,9 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
-; CHECK-NEXT: v_accvgpr_read_b32 v48, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v50, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v51, a53
+; CHECK-NEXT: v_accvgpr_read_b32 v52, a54
+; CHECK-NEXT: v_accvgpr_read_b32 v54, a56
+; CHECK-NEXT: v_accvgpr_read_b32 v55, a57
; CHECK-NEXT: v_accvgpr_read_b32 v30, a26
; CHECK-NEXT: v_accvgpr_read_b32 v25, a21
; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
@@ -1421,63 +1174,59 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v33, a27
; CHECK-NEXT: v_accvgpr_read_b32 v27, a59
; CHECK-NEXT: v_accvgpr_read_b32 v34, a26
-; CHECK-NEXT: v_cndmask_b32_e64 v48, v24, v28, s[42:43]
-; CHECK-NEXT: v_cndmask_b32_e64 v51, v27, v33, s[26:27]
-; CHECK-NEXT: v_cndmask_b32_e64 v50, v26, v34, s[26:27]
+; CHECK-NEXT: v_cndmask_b32_e64 v52, v24, v28, s[42:43]
+; CHECK-NEXT: v_cndmask_b32_e64 v55, v27, v33, s[26:27]
+; CHECK-NEXT: v_cndmask_b32_e64 v54, v26, v34, s[26:27]
; CHECK-NEXT: v_accvgpr_read_b32 v31, a23
; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v53, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
-; CHECK-NEXT: v_cndmask_b32_e64 v53, v21, v29, s[28:29]
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v41, a43
; CHECK-NEXT: v_accvgpr_read_b32 v32, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
; CHECK-NEXT: v_accvgpr_read_b32 v30, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a21
; CHECK-NEXT: v_accvgpr_read_b32 v28, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v52, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v54, a56
-; CHECK-NEXT: v_accvgpr_read_b32 v55, a57
+; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
+; CHECK-NEXT: v_cndmask_b32_e64 v41, v21, v29, s[28:29]
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a20
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a37
+; CHECK-NEXT: v_accvgpr_read_b32 v40, a42
; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
; CHECK-NEXT: v_accvgpr_read_b32 v31, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a38
+; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v40, a42
-; CHECK-NEXT: v_accvgpr_read_b32 v41, a43
+; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
+; CHECK-NEXT: v_cndmask_b32_e64 v40, v20, v32, s[28:29]
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a23
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a19
; CHECK-NEXT: v_accvgpr_read_b32 v42, a44
; CHECK-NEXT: v_accvgpr_read_b32 v43, a45
+; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
+; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
+; CHECK-NEXT: v_accvgpr_read_b32 v44, a46
+; CHECK-NEXT: v_accvgpr_read_b32 v45, a47
+; CHECK-NEXT: v_accvgpr_read_b32 v46, a48
+; CHECK-NEXT: v_accvgpr_read_b32 v47, a49
; CHECK-NEXT: v_accvgpr_read_b32 v27, a19
; CHECK-NEXT: v_accvgpr_read_b32 v26, a18
; CHECK-NEXT: v_accvgpr_read_b32 v25, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_cndmask_b32_e64 v52, v20, v24, s[28:29]
-; CHECK-NEXT: v_cndmask_b32_e64 v55, v23, v33, s[22:23]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a16
; CHECK-NEXT: v_accvgpr_read_b32 v29, a17
; CHECK-NEXT: v_accvgpr_read_b32 v28, a16
-; CHECK-NEXT: v_cndmask_b32_e64 v54, v22, v34, s[22:23]
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a19
+; CHECK-NEXT: v_cndmask_b32_e64 v43, v23, v35, s[22:23]
+; CHECK-NEXT: v_cndmask_b32_e64 v42, v22, v36, s[22:23]
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a17
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
+; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a19
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a18
; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
; CHECK-NEXT: v_accvgpr_read_b32 v2, a2
; CHECK-NEXT: v_accvgpr_read_b32 v3, a3
@@ -1493,138 +1242,134 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v13, a13
; CHECK-NEXT: v_accvgpr_read_b32 v14, a14
; CHECK-NEXT: v_accvgpr_read_b32 v15, a15
-; CHECK-NEXT: v_cndmask_b32_e64 v41, v17, v29, s[24:25]
-; CHECK-NEXT: v_cndmask_b32_e64 v40, v16, v20, s[24:25]
-; CHECK-NEXT: v_cndmask_b32_e64 v43, v19, v33, s[18:19]
-; CHECK-NEXT: v_cndmask_b32_e64 v42, v18, v34, s[18:19]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a15
+; CHECK-NEXT: v_cndmask_b32_e64 v45, v17, v31, s[24:25]
+; CHECK-NEXT: v_cndmask_b32_e64 v44, v16, v32, s[24:25]
+; CHECK-NEXT: v_cndmask_b32_e64 v47, v19, v35, s[18:19]
+; CHECK-NEXT: v_cndmask_b32_e64 v46, v18, v36, s[18:19]
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a15
; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a14
; CHECK-NEXT: v_accvgpr_read_b32 v12, a44
; CHECK-NEXT: v_accvgpr_read_b32 v13, a45
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a13
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a13
; CHECK-NEXT: v_accvgpr_read_b32 v30, a12
-; CHECK-NEXT: v_cndmask_b32_e64 v17, v13, v29, s[20:21]
-; CHECK-NEXT: v_cndmask_b32_e64 v16, v12, v30, s[20:21]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a10
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a8
+; CHECK-NEXT: v_accvgpr_read_b32 v25, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a6
+; CHECK-NEXT: v_accvgpr_read_b32 v23, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v22, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v19, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v18, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a12
+; CHECK-NEXT: v_cndmask_b32_e64 v19, v13, v31, s[20:21]
+; CHECK-NEXT: v_cndmask_b32_e64 v18, v12, v32, s[20:21]
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a11
; CHECK-NEXT: v_accvgpr_read_b32 v9, a41
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a15
-; CHECK-NEXT: v_cndmask_b32_e64 v21, v9, v29, s[16:17]
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a10
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a9
; CHECK-NEXT: v_accvgpr_read_b32 v30, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v15, a47
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a0
; CHECK-NEXT: v_accvgpr_read_b32 v29, a7
; CHECK-NEXT: v_accvgpr_read_b32 v28, a6
; CHECK-NEXT: v_accvgpr_read_b32 v27, a5
; CHECK-NEXT: v_accvgpr_read_b32 v26, a4
; CHECK-NEXT: v_accvgpr_read_b32 v25, a3
; CHECK-NEXT: v_accvgpr_read_b32 v24, a2
-; CHECK-NEXT: v_cndmask_b32_e64 v19, v15, v33, s[14:15]
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a11
-; CHECK-NEXT: v_cndmask_b32_e64 v20, v8, v30, s[16:17]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a37
-; CHECK-NEXT: v_accvgpr_read_b32 v11, a43
; CHECK-NEXT: v_accvgpr_read_b32 v23, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a10
+; CHECK-NEXT: v_cndmask_b32_e64 v23, v9, v31, s[16:17]
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a8
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a40
+; CHECK-NEXT: v_accvgpr_read_b32 v22, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a7
; CHECK-NEXT: v_accvgpr_read_b32 v30, a6
; CHECK-NEXT: v_accvgpr_read_b32 v29, a5
; CHECK-NEXT: v_accvgpr_read_b32 v28, a4
; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a1
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v11, v33, s[10:11]
-; CHECK-NEXT: v_cndmask_b32_e64 v25, v5, v29, s[12:13]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a4
+; CHECK-NEXT: v_cndmask_b32_e64 v22, v8, v32, s[16:17]
; CHECK-NEXT: v_accvgpr_read_b32 v33, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a37
; CHECK-NEXT: v_accvgpr_read_b32 v32, a6
-; CHECK-NEXT: v_cndmask_b32_e64 v24, v4, v30, s[12:13]
-; CHECK-NEXT: v_cndmask_b32_e64 v27, v7, v33, s[6:7]
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a4
; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a1
+; CHECK-NEXT: v_cndmask_b32_e64 v27, v5, v31, s[12:13]
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a36
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a3
; CHECK-NEXT: v_accvgpr_read_b32 v30, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a15
+; CHECK-NEXT: v_cndmask_b32_e64 v26, v4, v32, s[12:13]
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a3
; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_cndmask_b32_e64 v29, v1, v29, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v28, v0, v28, s[4:5]
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v14, a46
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v15, a47
+; CHECK-NEXT: v_accvgpr_read_b32 v21, a3
; CHECK-NEXT: v_accvgpr_read_b32 v34, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v14, a46
+; CHECK-NEXT: v_accvgpr_read_b32 v20, a2
+; CHECK-NEXT: v_cndmask_b32_e64 v21, v15, v35, s[14:15]
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
; CHECK-NEXT: v_accvgpr_read_b32 v2, a34
; CHECK-NEXT: v_accvgpr_read_b32 v3, a35
; CHECK-NEXT: v_accvgpr_read_b32 v10, a42
-; CHECK-NEXT: v_cndmask_b32_e64 v18, v14, v34, s[14:15]
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v11, a43
+; CHECK-NEXT: v_cndmask_b32_e64 v20, v14, v36, s[14:15]
+; CHECK-NEXT: v_accvgpr_read_b32 v25, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a0
; CHECK-NEXT: v_accvgpr_read_b32 v34, a10
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a10
; CHECK-NEXT: v_accvgpr_read_b32 v31, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a0
; CHECK-NEXT: v_accvgpr_read_b32 v6, a38
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v10, v34, s[10:11]
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a6
-; CHECK-NEXT: v_cndmask_b32_e64 v31, v3, v33, s[8:9]
-; CHECK-NEXT: v_cndmask_b32_e64 v30, v2, v30, s[8:9]
-; CHECK-NEXT: v_cndmask_b32_e64 v26, v6, v34, s[6:7]
+; CHECK-NEXT: v_accvgpr_read_b32 v7, a39
+; CHECK-NEXT: v_cndmask_b32_e64 v25, v11, v35, s[10:11]
+; CHECK-NEXT: v_cndmask_b32_e64 v24, v10, v36, s[10:11]
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a6
+; CHECK-NEXT: v_cndmask_b32_e64 v33, v3, v33, s[8:9]
+; CHECK-NEXT: v_cndmask_b32_e64 v32, v2, v32, s[8:9]
+; CHECK-NEXT: v_cndmask_b32_e64 v31, v1, v9, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v30, v0, v8, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v29, v7, v35, s[6:7]
+; CHECK-NEXT: v_cndmask_b32_e64 v28, v6, v36, s[6:7]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:16
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:32
-; CHECK-NEXT: v_accvgpr_read_b32 v35, a37
-; CHECK-NEXT: v_add_co_u32_e32 v20, vcc, 48, v0
-; CHECK-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x70, v0
-; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; CHECK-NEXT: v_accvgpr_read_b32 v44, a46
-; CHECK-NEXT: v_accvgpr_read_b32 v45, a47
-; CHECK-NEXT: v_accvgpr_read_b32 v46, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v47, a49
+; CHECK-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
+; CHECK-NEXT: flat_store_dwordx4 v[38:39], v[26:29] offset:16
+; CHECK-NEXT: flat_store_dwordx4 v[38:39], v[22:25] offset:32
+; CHECK-NEXT: v_accvgpr_read_b32 v37, a39
+; CHECK-NEXT: v_add_co_u32_e32 v22, vcc, 48, v38
+; CHECK-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v39, vcc
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x70, v38
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v39, vcc
; CHECK-NEXT: v_accvgpr_read_b32 v56, a58
; CHECK-NEXT: v_accvgpr_read_b32 v57, a59
; CHECK-NEXT: v_accvgpr_read_b32 v58, a60
; CHECK-NEXT: v_accvgpr_read_b32 v60, a62
; CHECK-NEXT: v_accvgpr_read_b32 v61, a63
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a2
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[16:19]
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[40:43] offset:64
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[52:55] offset:32
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[48:51] offset:48
-; CHECK-NEXT: flat_store_dwordx4 v[4:5], v[36:39]
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a6
+; CHECK-NEXT: flat_store_dwordx4 v[22:23], v[18:21]
+; CHECK-NEXT: flat_store_dwordx4 v[38:39], v[44:47] offset:64
+; CHECK-NEXT: flat_store_dwordx4 v[22:23], v[40:43] offset:32
+; CHECK-NEXT: flat_store_dwordx4 v[22:23], v[52:55] offset:48
+; CHECK-NEXT: flat_store_dwordx4 v[4:5], v[48:51]
; CHECK-NEXT: .LBB0_49: ; %UnifiedReturnBlock
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
@@ -1691,7 +1436,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_readlane_b32 s35, v63, 1
; CHECK-NEXT: v_readlane_b32 s34, v63, 0
; CHECK-NEXT: s_or_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:1632 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:1312 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index fa2ea184f2f83..bdff8c2766379 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -13,22 +13,23 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
; GCN-NEXT: s_mov_b32 s8, exec_lo
; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: flat_load_dwordx2 v[4:5], v[6:7]
-; GCN-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
+; GCN-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN-NEXT: s_mov_b32 s9, s8
; GCN-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GCN-NEXT: ; => This Inner Loop Header: Depth=2
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s4, v2
-; GCN-NEXT: v_readfirstlane_b32 s5, v3
-; GCN-NEXT: v_readfirstlane_b32 s6, v4
-; GCN-NEXT: v_readfirstlane_b32 s7, v5
+; GCN-NEXT: v_readfirstlane_b32 s4, v4
+; GCN-NEXT: v_readfirstlane_b32 s5, v5
+; GCN-NEXT: v_readfirstlane_b32 s6, v2
+; GCN-NEXT: v_readfirstlane_b32 s7, v3
; GCN-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
-; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 9209ba2fab3fd..d28675c72476f 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -583,7 +583,7 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR killed %11, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s64) from %ir.idx, addrspace 1)
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[GLOBAL_LOAD_DWORDX2_SADDR]], 16, 0, implicit $exec :: (invariant load (s128) from %ir.3 + 16, addrspace 4)
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_1:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[GLOBAL_LOAD_DWORDX2_SADDR]], 0, 0, implicit $exec :: (invariant load (s128) from %ir.3, align 32, addrspace 4)
- ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE killed [[GLOBAL_LOAD_DWORDX4_1]].sub0, %subreg.sub0, [[GLOBAL_LOAD_DWORDX4_1]].sub1, %subreg.sub1, [[GLOBAL_LOAD_DWORDX4_1]].sub2, %subreg.sub2, [[GLOBAL_LOAD_DWORDX4_1]].sub3, %subreg.sub3, killed [[GLOBAL_LOAD_DWORDX4_]].sub0, %subreg.sub4, [[GLOBAL_LOAD_DWORDX4_]].sub1, %subreg.sub5, [[GLOBAL_LOAD_DWORDX4_]].sub2, %subreg.sub6, [[GLOBAL_LOAD_DWORDX4_]].sub3, %subreg.sub7
+ ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE [[GLOBAL_LOAD_DWORDX4_1]].sub0, %subreg.sub0, [[GLOBAL_LOAD_DWORDX4_1]].sub1, %subreg.sub1, [[GLOBAL_LOAD_DWORDX4_1]].sub2, %subreg.sub2, [[GLOBAL_LOAD_DWORDX4_1]].sub3, %subreg.sub3, [[GLOBAL_LOAD_DWORDX4_]].sub0, %subreg.sub4, [[GLOBAL_LOAD_DWORDX4_]].sub1, %subreg.sub5, [[GLOBAL_LOAD_DWORDX4_]].sub2, %subreg.sub6, [[GLOBAL_LOAD_DWORDX4_]].sub3, %subreg.sub7
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_2:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 killed [[GLOBAL_LOAD_DWORDX2_SADDR]], 48, 0, implicit $exec :: (invariant load (s128) from %ir.add.ptr.i, addrspace 4)
; SI-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: {{ $}}
@@ -591,17 +591,17 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: successors: %bb.3(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.1, %38, %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_1]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_1]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_1]].sub2, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_1]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; SI-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_]].sub2, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
; SI-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3, killed [[V_READFIRSTLANE_B32_4]], %subreg.sub4, killed [[V_READFIRSTLANE_B32_5]], %subreg.sub5, killed [[V_READFIRSTLANE_B32_6]], %subreg.sub6, killed [[V_READFIRSTLANE_B32_7]], %subreg.sub7
; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
diff --git a/llvm/test/CodeGen/Thumb2/mve-clmul.ll b/llvm/test/CodeGen/Thumb2/mve-clmul.ll
index 0c9f1bfbd14cb..bebb9b3205cb5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-clmul.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-clmul.ll
@@ -6246,18 +6246,18 @@ define <4 x i64> @clmul_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
; CHECK-NEXT: sub sp, #496
; CHECK-NEXT: vmov q2, q0
; CHECK-NEXT: movs r3, #0
-; CHECK-NEXT: vmov.f32 s2, s9
+; CHECK-NEXT: vmov r12, s9
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: vmov r0, s8
; CHECK-NEXT: vmov q6, q1
; CHECK-NEXT: vmov.f32 s12, s24
; CHECK-NEXT: mov.w lr, #0
; CHECK-NEXT: vmov.f32 s14, s25
-; CHECK-NEXT: vmov r12, s2
-; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: lsll r4, r5, #1
+; CHECK-NEXT: vmov.f32 s2, s9
; CHECK-NEXT: mov r2, r12
+; CHECK-NEXT: mov r4, r0
; CHECK-NEXT: lsll r2, r3, #1
+; CHECK-NEXT: lsll r4, r5, #1
; CHECK-NEXT: vmov q4[2], q4[0], r4, r2
; CHECK-NEXT: adr.w r2, .LCPI22_32
; CHECK-NEXT: vldrw.u32 q1, [r2]
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
index e4a75cfbead0d..ab9ffda845434 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
@@ -4,8 +4,8 @@
define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
; CHECK-LABEL: loads_i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT: push {r4, r5, r6, r7, lr}
+; CHECK-NEXT: .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, lr}
; CHECK-NEXT: vldrw.u32 q3, [r1]
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: vmov.i64 q2, #0xffffffff
@@ -22,31 +22,30 @@ define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
; CHECK-NEXT: adds.w r12, r0, r1
; CHECK-NEXT: vmov r0, s7
; CHECK-NEXT: adc.w r1, r6, r5
-; CHECK-NEXT: asrs r5, r0, #31
-; CHECK-NEXT: adds r0, r0, r4
+; CHECK-NEXT: adds.w r8, r0, r4
+; CHECK-NEXT: vmov r4, r6, d4
+; CHECK-NEXT: asr.w r5, r0, #31
; CHECK-NEXT: adcs r5, r3
; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r4, r6, d4
; CHECK-NEXT: vldrw.u32 q1, [r2]
; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: adds r4, r4, r3
; CHECK-NEXT: asr.w r7, r3, #31
; CHECK-NEXT: adc.w r3, r7, r6
-; CHECK-NEXT: asr.w r7, lr, #31
; CHECK-NEXT: asrl r4, r3, r2
+; CHECK-NEXT: asr.w r2, lr, #31
; CHECK-NEXT: vmov r3, r6, d0
-; CHECK-NEXT: adds.w r2, lr, r3
-; CHECK-NEXT: adc.w r3, r7, r6
-; CHECK-NEXT: vmov r7, s4
-; CHECK-NEXT: asrl r2, r3, r7
-; CHECK-NEXT: vmov r3, s7
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r4
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: asrl r0, r5, r3
-; CHECK-NEXT: asrl r12, r1, r2
-; CHECK-NEXT: vmov q0[3], q0[1], r12, r0
-; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT: adds.w r0, lr, r3
+; CHECK-NEXT: adc.w r3, r2, r6
+; CHECK-NEXT: vmov r2, s4
+; CHECK-NEXT: asrl r0, r3, r2
+; CHECK-NEXT: vmov r2, s7
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT: vmov r0, s5
+; CHECK-NEXT: asrl r8, r5, r2
+; CHECK-NEXT: asrl r12, r1, r0
+; CHECK-NEXT: vmov q0[3], q0[1], r12, r8
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, pc}
entry:
%a = load <4 x i32>, ptr %A, align 4
%b = load <4 x i32>, ptr %B, align 4
@@ -138,11 +137,11 @@ define arm_aapcs_vfpcc void @load_store_i32(ptr %A, ptr %B, ptr %C, ptr %D) {
; CHECK-NEXT: vand q2, q2, q0
; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: vmov r1, r12, d5
-; CHECK-NEXT: vand q0, q1, q0
+; CHECK-NEXT: vand q1, q1, q0
; CHECK-NEXT: vmov lr, r4, d4
; CHECK-NEXT: vldrw.u32 q2, [r0]
-; CHECK-NEXT: vldrw.u32 q1, [r2]
-; CHECK-NEXT: vmov r9, r8, d0
+; CHECK-NEXT: vldrw.u32 q0, [r2]
+; CHECK-NEXT: vmov r9, r8, d2
; CHECK-NEXT: vmov r0, s11
; CHECK-NEXT: asrs r2, r0, #31
; CHECK-NEXT: adds r0, r0, r1
@@ -152,25 +151,24 @@ define arm_aapcs_vfpcc void @load_store_i32(ptr %A, ptr %B, ptr %C, ptr %D) {
; CHECK-NEXT: adds.w r2, r2, lr
; CHECK-NEXT: adc.w r7, r6, r4
; CHECK-NEXT: vmov r4, r5, d4
-; CHECK-NEXT: vmov r6, s6
+; CHECK-NEXT: vmov r6, s2
; CHECK-NEXT: asrl r2, r7, r6
-; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: adds.w r6, r4, r9
; CHECK-NEXT: asr.w r7, r4, #31
-; CHECK-NEXT: vmov r4, s4
+; CHECK-NEXT: vmov r4, s0
; CHECK-NEXT: adc.w r7, r7, r8
; CHECK-NEXT: asrl r6, r7, r4
-; CHECK-NEXT: vmov r4, r1, d1
-; CHECK-NEXT: vmov q0[2], q0[0], r6, r2
+; CHECK-NEXT: vmov r4, r1, d3
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r2
; CHECK-NEXT: asrs r2, r5, #31
-; CHECK-NEXT: vmov r7, s7
+; CHECK-NEXT: vmov r7, s3
; CHECK-NEXT: asrl r0, r11, r7
; CHECK-NEXT: adds r6, r5, r4
; CHECK-NEXT: adcs r1, r2
-; CHECK-NEXT: vmov r2, s6
+; CHECK-NEXT: vmov r2, s1
; CHECK-NEXT: asrl r6, r1, r2
-; CHECK-NEXT: vmov q0[3], q0[1], r6, r0
-; CHECK-NEXT: vstrw.32 q0, [r3]
+; CHECK-NEXT: vmov q1[3], q1[1], r6, r0
+; CHECK-NEXT: vstrw.32 q1, [r3]
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r11, pc}
entry:
%a = load <4 x i32>, ptr %A, align 4
@@ -347,13 +345,11 @@ define arm_aapcs_vfpcc void @mul_i32(ptr %A, ptr %B, i64 %C, ptr %D) {
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: vldrw.u32 q0, [r1]
; CHECK-NEXT: ldr.w lr, [sp, #20]
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vmov.f32 s14, s5
+; CHECK-NEXT: vmov r0, s1
+; CHECK-NEXT: vmov r1, s5
; CHECK-NEXT: vmov r5, s4
; CHECK-NEXT: vmov.f32 s4, s6
; CHECK-NEXT: vmov.f32 s6, s7
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov r1, s14
; CHECK-NEXT: smull r12, r3, r1, r0
; CHECK-NEXT: vmov r0, s0
; CHECK-NEXT: vmov.f32 s0, s2
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
index d9ed7fe548c5e..fab35554cd3ce 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
@@ -64,17 +64,15 @@ entry:
define arm_aapcs_vfpcc <4 x i32> @ext_add_trunc_i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: ext_add_trunc_i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov r0, s6
+; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov r2, s3
-; CHECK-NEXT: vmov.f32 s2, s5
+; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: add.w r12, r1, r0
; CHECK-NEXT: vmov r1, s7
; CHECK-NEXT: vmov r0, s0
; CHECK-NEXT: add r1, r2
-; CHECK-NEXT: vmov r2, s2
-; CHECK-NEXT: vmov.f32 s2, s1
-; CHECK-NEXT: vmov r3, s2
+; CHECK-NEXT: vmov r2, s5
; CHECK-NEXT: add r2, r3
; CHECK-NEXT: vmov r3, s4
; CHECK-NEXT: add r0, r3
@@ -184,9 +182,9 @@ define arm_aapcs_vfpcc <4 x i32> @ext_add_ashr_trunc_i32(<4 x i32> %a, <4 x i32>
; CHECK-NEXT: vand q3, q3, q2
; CHECK-NEXT: vmov r2, s2
; CHECK-NEXT: vmov r0, r1, d6
+; CHECK-NEXT: vmov lr, r12, d7
; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: vand q1, q1, q2
-; CHECK-NEXT: vmov lr, r12, d7
; CHECK-NEXT: vmov r3, r7, d2
; CHECK-NEXT: asrs r5, r2, #31
; CHECK-NEXT: adds r2, r2, r0
@@ -198,19 +196,19 @@ define arm_aapcs_vfpcc <4 x i32> @ext_add_ashr_trunc_i32(<4 x i32> %a, <4 x i32>
; CHECK-NEXT: adds.w r0, r0, lr
; CHECK-NEXT: adc.w r1, r1, r12
; CHECK-NEXT: asrs r4, r5, #31
+; CHECK-NEXT: asrl r0, r1, #1
; CHECK-NEXT: adds r6, r5, r3
; CHECK-NEXT: vmov r3, r5, d3
-; CHECK-NEXT: vmov.f32 s6, s1
-; CHECK-NEXT: asrl r0, r1, #1
; CHECK-NEXT: adcs r7, r4
+; CHECK-NEXT: vmov r1, s1
; CHECK-NEXT: asrl r6, r7, #1
-; CHECK-NEXT: vmov q0[2], q0[0], r6, r2
-; CHECK-NEXT: vmov r1, s6
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r2
; CHECK-NEXT: adds r6, r1, r3
; CHECK-NEXT: asr.w r2, r1, #31
; CHECK-NEXT: adc.w r1, r2, r5
; CHECK-NEXT: asrl r6, r1, #1
-; CHECK-NEXT: vmov q0[3], q0[1], r6, r0
+; CHECK-NEXT: vmov q1[3], q1[1], r6, r0
+; CHECK-NEXT: vmov q0, q1
; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
entry:
%sa = sext <4 x i32> %a to <4 x i64>
@@ -312,7 +310,6 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b)
; CHECK-NEXT: sbc lr, r3, #0
; CHECK-NEXT: vmov r3, s6
; CHECK-NEXT: umull r0, r8, r0, r2
-; CHECK-NEXT: vmov.f32 s2, s1
; CHECK-NEXT: vmov.f32 s6, s5
; CHECK-NEXT: adds r4, r6, r3
; CHECK-NEXT: asr.w r7, r6, #31
@@ -337,11 +334,11 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b)
; CHECK-NEXT: rsb.w r1, r10, #0
; CHECK-NEXT: lsll r4, r5, r3
; CHECK-NEXT: lsll r0, r7, r1
-; CHECK-NEXT: vmov r1, s6
+; CHECK-NEXT: vmov r1, s5
; CHECK-NEXT: lsll r0, r7, r2
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: vmov q3[2], q3[0], r4, r0
-; CHECK-NEXT: vmov r0, s2
+; CHECK-NEXT: vmov r0, s1
; CHECK-NEXT: vmov q3[3], q3[1], r5, r7
; CHECK-NEXT: vpsel q2, q3, q2
; CHECK-NEXT: adds r4, r0, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index 40d534a0ebfc5..d441c33712520 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -235,22 +235,20 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(ptr nocapture readonly %pSrcA, ptr no
; CHECK-NEXT: mov.w r2, #-1
; CHECK-NEXT: vmov.f32 s16, s10
; CHECK-NEXT: str r5, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT: vmov.f32 s20, s14
-; CHECK-NEXT: mov.w r8, #0
; CHECK-NEXT: vmov.f32 s18, s11
+; CHECK-NEXT: mov.w r8, #0
+; CHECK-NEXT: vmov.f32 s20, s14
; CHECK-NEXT: vmov.f32 s22, s15
; CHECK-NEXT: vmullb.s32 q6, q5, q4
-; CHECK-NEXT: vmov.f32 s14, s13
+; CHECK-NEXT: vmov r6, s13
; CHECK-NEXT: vmov r4, r7, d12
; CHECK-NEXT: asrl r4, r7, #31
-; CHECK-NEXT: vmov.f32 s10, s9
; CHECK-NEXT: rsbs.w r5, r4, #-2147483648
; CHECK-NEXT: sbcs.w r5, r2, r7
; CHECK-NEXT: csetm r5, lt
; CHECK-NEXT: bfi r8, r5, #0, #8
; CHECK-NEXT: vmov r10, r5, d13
; CHECK-NEXT: asrl r10, r5, #31
-; CHECK-NEXT: vmov r6, s14
; CHECK-NEXT: rsbs.w r3, r10, #-2147483648
; CHECK-NEXT: vmov q4[2], q4[0], r4, r10
; CHECK-NEXT: sbcs.w r3, r2, r5
@@ -282,7 +280,7 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(ptr nocapture readonly %pSrcA, ptr no
; CHECK-NEXT: sbcs.w r3, r2, r7
; CHECK-NEXT: csetm r3, lt
; CHECK-NEXT: bfi r5, r3, #0, #8
-; CHECK-NEXT: vmov r3, s10
+; CHECK-NEXT: vmov r3, s9
; CHECK-NEXT: smull r6, r3, r6, r3
; CHECK-NEXT: asrl r6, r3, #31
; CHECK-NEXT: rsbs.w r1, r6, #-2147483648
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
index 16a51b2027d5e..32e1adc0ea4fa 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
@@ -522,28 +522,29 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3step_f16(<32 x half> %src) {
; CHECKFP: @ %bb.0: @ %entry
; CHECKFP-NEXT: .vsave {d8, d9}
; CHECKFP-NEXT: vpush {d8, d9}
+; CHECKFP-NEXT: vmovx.f16 s14, s2
+; CHECKFP-NEXT: vmov.f32 s12, s1
+; CHECKFP-NEXT: vins.f16 s12, s14
; CHECKFP-NEXT: vmov.f32 s13, s4
+; CHECKFP-NEXT: vmovx.f16 s14, s5
; CHECKFP-NEXT: vmovx.f16 s4, s4
+; CHECKFP-NEXT: vins.f16 s13, s14
+; CHECKFP-NEXT: vmov.f32 s14, s7
; CHECKFP-NEXT: vmovx.f16 s17, s3
; CHECKFP-NEXT: vins.f16 s3, s4
-; CHECKFP-NEXT: vmovx.f16 s4, s7
+; CHECKFP-NEXT: vmovx.f16 s4, s14
; CHECKFP-NEXT: vmovx.f16 s18, s6
; CHECKFP-NEXT: vmovx.f16 s16, s0
; CHECKFP-NEXT: vins.f16 s6, s4
-; CHECKFP-NEXT: vmovx.f16 s14, s2
-; CHECKFP-NEXT: vmov.f32 s12, s1
; CHECKFP-NEXT: vmovx.f16 s4, s10
; CHECKFP-NEXT: vmovx.f16 s19, s9
-; CHECKFP-NEXT: vins.f16 s12, s14
-; CHECKFP-NEXT: vmovx.f16 s14, s5
-; CHECKFP-NEXT: vins.f16 s16, s2
-; CHECKFP-NEXT: vmovx.f16 s2, s11
; CHECKFP-NEXT: vmovx.f16 s15, s8
; CHECKFP-NEXT: vins.f16 s18, s8
; CHECKFP-NEXT: vmovx.f16 s8, s1
+; CHECKFP-NEXT: vins.f16 s16, s2
+; CHECKFP-NEXT: vmovx.f16 s2, s11
; CHECKFP-NEXT: vins.f16 s9, s4
-; CHECKFP-NEXT: vins.f16 s13, s14
-; CHECKFP-NEXT: vmov.f32 s14, s7
+; CHECKFP-NEXT: vins.f16 s7, s15
; CHECKFP-NEXT: vins.f16 s10, s2
; CHECKFP-NEXT: vmov.f32 s1, s3
; CHECKFP-NEXT: vins.f16 s19, s11
@@ -551,9 +552,9 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3step_f16(<32 x half> %src) {
; CHECKFP-NEXT: vins.f16 s0, s8
; CHECKFP-NEXT: vmov.f32 s2, s6
; CHECKFP-NEXT: vmov.f32 s3, s9
-; CHECKFP-NEXT: vins.f16 s14, s15
-; CHECKFP-NEXT: vmov.f32 s15, s10
+; CHECKFP-NEXT: vmov.f32 s14, s7
; CHECKFP-NEXT: vadd.f16 q0, q0, q4
+; CHECKFP-NEXT: vmov.f32 s15, s10
; CHECKFP-NEXT: vadd.f16 q0, q0, q3
; CHECKFP-NEXT: vpop {d8, d9}
; CHECKFP-NEXT: bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index d01734e3c1358..3a2c3a1bc04a9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -366,34 +366,35 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3step_i16(<32 x i16> %src) {
; CHECK-NEXT: vmov.f32 s16, s1
; CHECK-NEXT: vins.f16 s12, s2
; CHECK-NEXT: vmovx.f16 s2, s2
+; CHECK-NEXT: vmovx.f16 s14, s6
; CHECK-NEXT: vins.f16 s16, s2
-; CHECK-NEXT: vmovx.f16 s2, s5
; CHECK-NEXT: vmov.f32 s17, s4
-; CHECK-NEXT: vmovx.f16 s14, s6
-; CHECK-NEXT: vins.f16 s17, s2
-; CHECK-NEXT: vmovx.f16 s2, s8
-; CHECK-NEXT: vmov.f32 s18, s7
-; CHECK-NEXT: vins.f16 s14, s8
-; CHECK-NEXT: vins.f16 s18, s2
-; CHECK-NEXT: vmovx.f16 s2, s11
-; CHECK-NEXT: vmovx.f16 s8, s10
-; CHECK-NEXT: vins.f16 s10, s2
-; CHECK-NEXT: vmovx.f16 s2, s1
+; CHECK-NEXT: vmovx.f16 s2, s5
; CHECK-NEXT: vmovx.f16 s13, s3
-; CHECK-NEXT: vins.f16 s0, s2
-; CHECK-NEXT: vmovx.f16 s2, s4
-; CHECK-NEXT: vins.f16 s3, s2
-; CHECK-NEXT: vmovx.f16 s2, s7
+; CHECK-NEXT: vins.f16 s14, s8
+; CHECK-NEXT: vins.f16 s17, s2
+; CHECK-NEXT: vmov.f32 s2, s7
+; CHECK-NEXT: vmovx.f16 s8, s8
+; CHECK-NEXT: vins.f16 s13, s5
+; CHECK-NEXT: vins.f16 s7, s8
+; CHECK-NEXT: vmovx.f16 s4, s4
+; CHECK-NEXT: vmovx.f16 s2, s2
+; CHECK-NEXT: vmovx.f16 s8, s11
+; CHECK-NEXT: vmovx.f16 s5, s10
; CHECK-NEXT: vmovx.f16 s15, s9
+; CHECK-NEXT: vins.f16 s10, s8
; CHECK-NEXT: vins.f16 s6, s2
-; CHECK-NEXT: vins.f16 s9, s8
+; CHECK-NEXT: vmovx.f16 s8, s1
+; CHECK-NEXT: vins.f16 s3, s4
+; CHECK-NEXT: vins.f16 s9, s5
; CHECK-NEXT: vmov.f32 s1, s3
; CHECK-NEXT: vins.f16 s15, s11
-; CHECK-NEXT: vins.f16 s13, s5
+; CHECK-NEXT: vins.f16 s0, s8
; CHECK-NEXT: vmov.f32 s2, s6
; CHECK-NEXT: vmov.f32 s3, s9
-; CHECK-NEXT: vmov.f32 s19, s10
+; CHECK-NEXT: vmov.f32 s18, s7
; CHECK-NEXT: vadd.i16 q0, q0, q3
+; CHECK-NEXT: vmov.f32 s19, s10
; CHECK-NEXT: vadd.i16 q0, q0, q4
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: bx lr
@@ -1402,28 +1403,29 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3step_f16(<32 x half> %src) {
; CHECKFP: @ %bb.0: @ %entry
; CHECKFP-NEXT: .vsave {d8, d9}
; CHECKFP-NEXT: vpush {d8, d9}
+; CHECKFP-NEXT: vmovx.f16 s14, s2
+; CHECKFP-NEXT: vmov.f32 s12, s1
+; CHECKFP-NEXT: vins.f16 s12, s14
; CHECKFP-NEXT: vmov.f32 s13, s4
+; CHECKFP-NEXT: vmovx.f16 s14, s5
; CHECKFP-NEXT: vmovx.f16 s4, s4
+; CHECKFP-NEXT: vins.f16 s13, s14
+; CHECKFP-NEXT: vmov.f32 s14, s7
; CHECKFP-NEXT: vmovx.f16 s17, s3
; CHECKFP-NEXT: vins.f16 s3, s4
-; CHECKFP-NEXT: vmovx.f16 s4, s7
+; CHECKFP-NEXT: vmovx.f16 s4, s14
; CHECKFP-NEXT: vmovx.f16 s18, s6
; CHECKFP-NEXT: vmovx.f16 s16, s0
; CHECKFP-NEXT: vins.f16 s6, s4
-; CHECKFP-NEXT: vmovx.f16 s14, s2
-; CHECKFP-NEXT: vmov.f32 s12, s1
; CHECKFP-NEXT: vmovx.f16 s4, s10
; CHECKFP-NEXT: vmovx.f16 s19, s9
-; CHECKFP-NEXT: vins.f16 s12, s14
-; CHECKFP-NEXT: vmovx.f16 s14, s5
-; CHECKFP-NEXT: vins.f16 s16, s2
-; CHECKFP-NEXT: vmovx.f16 s2, s11
; CHECKFP-NEXT: vmovx.f16 s15, s8
; CHECKFP-NEXT: vins.f16 s18, s8
; CHECKFP-NEXT: vmovx.f16 s8, s1
+; CHECKFP-NEXT: vins.f16 s16, s2
+; CHECKFP-NEXT: vmovx.f16 s2, s11
; CHECKFP-NEXT: vins.f16 s9, s4
-; CHECKFP-NEXT: vins.f16 s13, s14
-; CHECKFP-NEXT: vmov.f32 s14, s7
+; CHECKFP-NEXT: vins.f16 s7, s15
; CHECKFP-NEXT: vins.f16 s10, s2
; CHECKFP-NEXT: vmov.f32 s1, s3
; CHECKFP-NEXT: vins.f16 s19, s11
@@ -1431,9 +1433,9 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3step_f16(<32 x half> %src) {
; CHECKFP-NEXT: vins.f16 s0, s8
; CHECKFP-NEXT: vmov.f32 s2, s6
; CHECKFP-NEXT: vmov.f32 s3, s9
-; CHECKFP-NEXT: vins.f16 s14, s15
-; CHECKFP-NEXT: vmov.f32 s15, s10
+; CHECKFP-NEXT: vmov.f32 s14, s7
; CHECKFP-NEXT: vadd.f16 q0, q0, q4
+; CHECKFP-NEXT: vmov.f32 s15, s10
; CHECKFP-NEXT: vadd.f16 q0, q0, q3
; CHECKFP-NEXT: vpop {d8, d9}
; CHECKFP-NEXT: bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
index 21f5439cecf94..ade56489c0855 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
@@ -551,8 +551,8 @@ define void @vabd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y,
; CHECK-NEXT: vmov q2[2], q2[0], r3, r12
; CHECK-NEXT: vmov r3, s23
; CHECK-NEXT: sub.w r12, r4, r3
-; CHECK-NEXT: vmov r4, s22
-; CHECK-NEXT: vmov r3, s26
+; CHECK-NEXT: vmov r4, s21
+; CHECK-NEXT: vmov r3, s25
; CHECK-NEXT: vand q5, q5, q0
; CHECK-NEXT: vand q6, q6, q0
; CHECK-NEXT: subs r3, r3, r4
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
index b919873c02bf9..d2f438a2a82e3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -282,46 +282,47 @@ entry:
define void @vld3_v8i16(ptr %src, ptr %dst) {
; CHECK-LABEL: vld3_v8i16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
+; CHECK-NEXT: .vsave {d8, d9, d10}
+; CHECK-NEXT: vpush {d8, d9, d10}
; CHECK-NEXT: vldrw.u32 q0, [r0]
; CHECK-NEXT: vldrw.u32 q2, [r0, #16]
; CHECK-NEXT: vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT: vmovx.f16 s6, s2
-; CHECK-NEXT: vmov.f32 s4, s1
-; CHECK-NEXT: vins.f16 s4, s6
-; CHECK-NEXT: vmovx.f16 s6, s9
-; CHECK-NEXT: vmov.f32 s5, s8
-; CHECK-NEXT: vmovx.f16 s7, s12
-; CHECK-NEXT: vins.f16 s5, s6
-; CHECK-NEXT: vmov.f32 s6, s11
-; CHECK-NEXT: vins.f16 s6, s7
-; CHECK-NEXT: vmovx.f16 s16, s15
-; CHECK-NEXT: vmov.f32 s7, s14
-; CHECK-NEXT: vmovx.f16 s17, s3
-; CHECK-NEXT: vins.f16 s7, s16
; CHECK-NEXT: vmovx.f16 s16, s0
+; CHECK-NEXT: vmovx.f16 s6, s2
; CHECK-NEXT: vins.f16 s16, s2
; CHECK-NEXT: vmovx.f16 s2, s1
+; CHECK-NEXT: vmov.f32 s4, s1
; CHECK-NEXT: vins.f16 s0, s2
+; CHECK-NEXT: vmov.f32 s20, s11
; CHECK-NEXT: vmovx.f16 s2, s8
+; CHECK-NEXT: vmovx.f16 s17, s3
; CHECK-NEXT: vins.f16 s3, s2
-; CHECK-NEXT: vmovx.f16 s2, s11
+; CHECK-NEXT: vins.f16 s4, s6
+; CHECK-NEXT: vmovx.f16 s6, s9
+; CHECK-NEXT: vmov.f32 s5, s8
+; CHECK-NEXT: vmovx.f16 s2, s20
; CHECK-NEXT: vmovx.f16 s18, s10
+; CHECK-NEXT: vins.f16 s5, s6
+; CHECK-NEXT: vmovx.f16 s6, s12
; CHECK-NEXT: vins.f16 s10, s2
; CHECK-NEXT: vmovx.f16 s2, s14
; CHECK-NEXT: vmovx.f16 s19, s13
+; CHECK-NEXT: vins.f16 s11, s6
; CHECK-NEXT: vins.f16 s13, s2
-; CHECK-NEXT: vmov.f32 s1, s3
+; CHECK-NEXT: vmov.f32 s6, s11
+; CHECK-NEXT: vmovx.f16 s11, s15
+; CHECK-NEXT: vmov.f32 s7, s14
; CHECK-NEXT: vins.f16 s18, s12
+; CHECK-NEXT: vmov.f32 s1, s3
; CHECK-NEXT: vins.f16 s19, s15
; CHECK-NEXT: vins.f16 s17, s9
; CHECK-NEXT: vmov.f32 s2, s10
; CHECK-NEXT: vmov.f32 s3, s13
+; CHECK-NEXT: vins.f16 s7, s11
; CHECK-NEXT: vadd.i16 q0, q0, q4
; CHECK-NEXT: vadd.i16 q0, q0, q1
; CHECK-NEXT: vstrw.32 q0, [r1]
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: vpop {d8, d9, d10}
; CHECK-NEXT: bx lr
entry:
%l1 = load <24 x i16>, ptr %src, align 4
@@ -342,48 +343,54 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
; CHECK-LV-NEXT: vldrw.u32 q0, [r0, #48]
; CHECK-LV-NEXT: vldrw.u32 q2, [r0, #64]
; CHECK-LV-NEXT: vldrw.u32 q3, [r0, #80]
-; CHECK-LV-NEXT: vmovx.f16 s6, s2
-; CHECK-LV-NEXT: vmov.f32 s4, s1
-; CHECK-LV-NEXT: vins.f16 s4, s6
-; CHECK-LV-NEXT: vmovx.f16 s6, s9
-; CHECK-LV-NEXT: vmov.f32 s5, s8
-; CHECK-LV-NEXT: vmovx.f16 s7, s12
-; CHECK-LV-NEXT: vins.f16 s5, s6
-; CHECK-LV-NEXT: vmov.f32 s6, s11
-; CHECK-LV-NEXT: vins.f16 s6, s7
-; CHECK-LV-NEXT: vmovx.f16 s16, s15
-; CHECK-LV-NEXT: vmov.f32 s7, s14
-; CHECK-LV-NEXT: vmovx.f16 s17, s3
-; CHECK-LV-NEXT: vins.f16 s7, s16
; CHECK-LV-NEXT: vmovx.f16 s16, s0
+; CHECK-LV-NEXT: vmovx.f16 s6, s2
; CHECK-LV-NEXT: vins.f16 s16, s2
; CHECK-LV-NEXT: vmovx.f16 s2, s1
+; CHECK-LV-NEXT: vmov.f32 s4, s1
; CHECK-LV-NEXT: vins.f16 s0, s2
+; CHECK-LV-NEXT: vmov.f32 s20, s11
; CHECK-LV-NEXT: vmovx.f16 s2, s8
+; CHECK-LV-NEXT: vmovx.f16 s17, s3
; CHECK-LV-NEXT: vins.f16 s3, s2
-; CHECK-LV-NEXT: vmovx.f16 s2, s11
+; CHECK-LV-NEXT: vins.f16 s4, s6
+; CHECK-LV-NEXT: vmovx.f16 s6, s9
+; CHECK-LV-NEXT: vmov.f32 s5, s8
+; CHECK-LV-NEXT: vmovx.f16 s2, s20
; CHECK-LV-NEXT: vmovx.f16 s18, s10
+; CHECK-LV-NEXT: vins.f16 s5, s6
+; CHECK-LV-NEXT: vmovx.f16 s6, s12
; CHECK-LV-NEXT: vins.f16 s10, s2
; CHECK-LV-NEXT: vmovx.f16 s2, s14
; CHECK-LV-NEXT: vmovx.f16 s19, s13
+; CHECK-LV-NEXT: vins.f16 s11, s6
; CHECK-LV-NEXT: vins.f16 s13, s2
-; CHECK-LV-NEXT: vmov.f32 s1, s3
+; CHECK-LV-NEXT: vmov.f32 s6, s11
+; CHECK-LV-NEXT: vmovx.f16 s11, s15
+; CHECK-LV-NEXT: vmov.f32 s7, s14
; CHECK-LV-NEXT: vins.f16 s18, s12
+; CHECK-LV-NEXT: vmov.f32 s1, s3
; CHECK-LV-NEXT: vins.f16 s19, s15
; CHECK-LV-NEXT: vins.f16 s17, s9
; CHECK-LV-NEXT: vmov.f32 s2, s10
; CHECK-LV-NEXT: vmov.f32 s3, s13
-; CHECK-LV-NEXT: vldrw.u32 q2, [r0, #16]
+; CHECK-LV-NEXT: vins.f16 s7, s11
; CHECK-LV-NEXT: vadd.i16 q0, q0, q4
; CHECK-LV-NEXT: vldrw.u32 q4, [r0]
+; CHECK-LV-NEXT: vldrw.u32 q2, [r0, #16]
; CHECK-LV-NEXT: vadd.i16 q0, q0, q1
; CHECK-LV-NEXT: vldrw.u32 q1, [r0, #32]
+; CHECK-LV-NEXT: vmovx.f16 s13, s19
; CHECK-LV-NEXT: vmovx.f16 s14, s10
-; CHECK-LV-NEXT: vmov.f32 s22, s11
+; CHECK-LV-NEXT: vins.f16 s13, s9
; CHECK-LV-NEXT: vins.f16 s14, s4
+; CHECK-LV-NEXT: vmovx.f16 s9, s9
+; CHECK-LV-NEXT: vmov.f32 s21, s8
; CHECK-LV-NEXT: vmovx.f16 s4, s4
; CHECK-LV-NEXT: vmovx.f16 s15, s5
-; CHECK-LV-NEXT: vins.f16 s22, s4
+; CHECK-LV-NEXT: vins.f16 s21, s9
+; CHECK-LV-NEXT: vmov.f32 s9, s11
+; CHECK-LV-NEXT: vins.f16 s11, s4
; CHECK-LV-NEXT: vmovx.f16 s4, s7
; CHECK-LV-NEXT: vins.f16 s15, s7
; CHECK-LV-NEXT: vmovx.f16 s7, s6
@@ -392,25 +399,21 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
; CHECK-LV-NEXT: vmovx.f16 s12, s16
; CHECK-LV-NEXT: vins.f16 s16, s4
; CHECK-LV-NEXT: vmovx.f16 s4, s8
-; CHECK-LV-NEXT: vmovx.f16 s13, s19
; CHECK-LV-NEXT: vins.f16 s19, s4
-; CHECK-LV-NEXT: vmovx.f16 s4, s11
+; CHECK-LV-NEXT: vmovx.f16 s4, s9
; CHECK-LV-NEXT: vins.f16 s12, s18
; CHECK-LV-NEXT: vmov.f32 s20, s17
; CHECK-LV-NEXT: vmovx.f16 s18, s18
; CHECK-LV-NEXT: vins.f16 s10, s4
; CHECK-LV-NEXT: vins.f16 s5, s7
-; CHECK-LV-NEXT: vins.f16 s13, s9
; CHECK-LV-NEXT: vins.f16 s20, s18
; CHECK-LV-NEXT: vmov.f32 s17, s19
-; CHECK-LV-NEXT: vmovx.f16 s9, s9
-; CHECK-LV-NEXT: vmov.f32 s21, s8
; CHECK-LV-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LV-NEXT: vmov.f32 s18, s10
-; CHECK-LV-NEXT: vins.f16 s21, s9
; CHECK-LV-NEXT: vmov.f32 s19, s5
; CHECK-LV-NEXT: vmov.f32 s23, s6
; CHECK-LV-NEXT: vadd.i16 q1, q4, q3
+; CHECK-LV-NEXT: vmov.f32 s22, s11
; CHECK-LV-NEXT: vadd.i16 q1, q1, q5
; CHECK-LV-NEXT: vstrw.32 q1, [r1]
; CHECK-LV-NEXT: vpop {d8, d9, d10, d11}
@@ -423,33 +426,33 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
; CHECK-LIS-NEXT: vldrw.u32 q0, [r0, #48]
; CHECK-LIS-NEXT: vldrw.u32 q2, [r0, #64]
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #80]
-; CHECK-LIS-NEXT: vmovx.f16 s6, s2
-; CHECK-LIS-NEXT: vmov.f32 s4, s1
-; CHECK-LIS-NEXT: vins.f16 s4, s6
-; CHECK-LIS-NEXT: vmovx.f16 s6, s9
-; CHECK-LIS-NEXT: vmov.f32 s5, s8
-; CHECK-LIS-NEXT: vmovx.f16 s7, s12
-; CHECK-LIS-NEXT: vins.f16 s5, s6
-; CHECK-LIS-NEXT: vmov.f32 s6, s11
-; CHECK-LIS-NEXT: vins.f16 s6, s7
-; CHECK-LIS-NEXT: vmovx.f16 s16, s15
-; CHECK-LIS-NEXT: vmov.f32 s7, s14
-; CHECK-LIS-NEXT: vmovx.f16 s17, s3
-; CHECK-LIS-NEXT: vins.f16 s7, s16
; CHECK-LIS-NEXT: vmovx.f16 s16, s0
+; CHECK-LIS-NEXT: vmovx.f16 s6, s2
; CHECK-LIS-NEXT: vins.f16 s16, s2
; CHECK-LIS-NEXT: vmovx.f16 s2, s1
+; CHECK-LIS-NEXT: vmov.f32 s4, s1
; CHECK-LIS-NEXT: vins.f16 s0, s2
+; CHECK-LIS-NEXT: vmov.f32 s20, s11
; CHECK-LIS-NEXT: vmovx.f16 s2, s8
+; CHECK-LIS-NEXT: vmovx.f16 s17, s3
; CHECK-LIS-NEXT: vins.f16 s3, s2
-; CHECK-LIS-NEXT: vmovx.f16 s2, s11
+; CHECK-LIS-NEXT: vins.f16 s4, s6
+; CHECK-LIS-NEXT: vmovx.f16 s6, s9
+; CHECK-LIS-NEXT: vmov.f32 s5, s8
+; CHECK-LIS-NEXT: vmovx.f16 s2, s20
; CHECK-LIS-NEXT: vmovx.f16 s18, s10
+; CHECK-LIS-NEXT: vins.f16 s5, s6
+; CHECK-LIS-NEXT: vmovx.f16 s6, s12
; CHECK-LIS-NEXT: vins.f16 s10, s2
; CHECK-LIS-NEXT: vmovx.f16 s2, s14
; CHECK-LIS-NEXT: vmovx.f16 s19, s13
+; CHECK-LIS-NEXT: vins.f16 s11, s6
; CHECK-LIS-NEXT: vins.f16 s13, s2
-; CHECK-LIS-NEXT: vmov.f32 s1, s3
+; CHECK-LIS-NEXT: vmov.f32 s6, s11
+; CHECK-LIS-NEXT: vmovx.f16 s11, s15
+; CHECK-LIS-NEXT: vmov.f32 s7, s14
; CHECK-LIS-NEXT: vins.f16 s18, s12
+; CHECK-LIS-NEXT: vmov.f32 s1, s3
; CHECK-LIS-NEXT: vins.f16 s19, s15
; CHECK-LIS-NEXT: vmov.f32 s3, s13
; CHECK-LIS-NEXT: vins.f16 s17, s9
@@ -457,14 +460,20 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #16]
; CHECK-LIS-NEXT: vadd.i16 q0, q0, q4
; CHECK-LIS-NEXT: vldrw.u32 q4, [r0]
+; CHECK-LIS-NEXT: vins.f16 s7, s11
+; CHECK-LIS-NEXT: vmovx.f16 s10, s14
; CHECK-LIS-NEXT: vadd.i16 q0, q0, q1
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0, #32]
-; CHECK-LIS-NEXT: vmovx.f16 s10, s14
-; CHECK-LIS-NEXT: vmov.f32 s22, s15
+; CHECK-LIS-NEXT: vmovx.f16 s9, s19
+; CHECK-LIS-NEXT: vmov.f32 s21, s12
; CHECK-LIS-NEXT: vins.f16 s10, s4
+; CHECK-LIS-NEXT: vins.f16 s9, s13
+; CHECK-LIS-NEXT: vmovx.f16 s13, s13
; CHECK-LIS-NEXT: vmovx.f16 s4, s4
; CHECK-LIS-NEXT: vmovx.f16 s11, s5
-; CHECK-LIS-NEXT: vins.f16 s22, s4
+; CHECK-LIS-NEXT: vins.f16 s21, s13
+; CHECK-LIS-NEXT: vmov.f32 s13, s15
+; CHECK-LIS-NEXT: vins.f16 s15, s4
; CHECK-LIS-NEXT: vmovx.f16 s4, s7
; CHECK-LIS-NEXT: vins.f16 s11, s7
; CHECK-LIS-NEXT: vmovx.f16 s7, s6
@@ -473,25 +482,21 @@ define void @vld3_v16i16(ptr %src, ptr %dst) {
; CHECK-LIS-NEXT: vmovx.f16 s8, s16
; CHECK-LIS-NEXT: vins.f16 s16, s4
; CHECK-LIS-NEXT: vmovx.f16 s4, s12
-; CHECK-LIS-NEXT: vmovx.f16 s9, s19
; CHECK-LIS-NEXT: vins.f16 s19, s4
-; CHECK-LIS-NEXT: vmovx.f16 s4, s15
+; CHECK-LIS-NEXT: vmovx.f16 s4, s13
; CHECK-LIS-NEXT: vins.f16 s8, s18
; CHECK-LIS-NEXT: vmov.f32 s20, s17
; CHECK-LIS-NEXT: vmovx.f16 s18, s18
; CHECK-LIS-NEXT: vins.f16 s14, s4
; CHECK-LIS-NEXT: vins.f16 s5, s7
-; CHECK-LIS-NEXT: vins.f16 s9, s13
; CHECK-LIS-NEXT: vins.f16 s20, s18
; CHECK-LIS-NEXT: vmov.f32 s17, s19
-; CHECK-LIS-NEXT: vmovx.f16 s13, s13
-; CHECK-LIS-NEXT: vmov.f32 s21, s12
; CHECK-LIS-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LIS-NEXT: vmov.f32 s18, s14
-; CHECK-LIS-NEXT: vins.f16 s21, s13
; CHECK-LIS-NEXT: vmov.f32 s19, s5
; CHECK-LIS-NEXT: vmov.f32 s23, s6
; CHECK-LIS-NEXT: vadd.i16 q1, q4, q2
+; CHECK-LIS-NEXT: vmov.f32 s22, s15
; CHECK-LIS-NEXT: vadd.i16 q1, q1, q5
; CHECK-LIS-NEXT: vstrw.32 q1, [r1]
; CHECK-LIS-NEXT: vpop {d8, d9, d10, d11}
>From 82fcc2cb56858516e2c2d6d3af06ac6a6c20013c Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 25 Jun 2026 10:49:46 -0400
Subject: [PATCH 3/6] [utils] Don't flag undef when replacing existing uses in
same hunk
The undef deprecation check was flagging check line changes that restructure
existing undef uses within the same diff hunk, but the intention
of the check is to prevent the introduction of *new* uses.
Check each diff hunk independently: only flag hunks that add undef
without also removing undef in the same hunk.
---
llvm/utils/git/code-format-helper.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 1e8c332554a8e..47a5b0f06efd9 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -426,9 +426,17 @@ def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str
undef_regex = r"(?<!%)\bundef\b"
else:
undef_regex = r"UndefValue::get"
- # search for additions of undef
- if re.search(r"^[+].*" + undef_regex, file, re.MULTILINE):
- files.append(filename)
+
+ # Split the file diff into hunks and search for additions
+ # of undef in each hunk.
+ hunks = re.split(r"^@@", file, flags=re.MULTILINE)
+ for hunk in hunks:
+ if re.findall(r"^[+].*" + undef_regex, hunk, re.MULTILINE):
+ # If there are removals of undef in this hunk, this is
+ # likely a legitimate transformation.
+ if not re.findall(r"^-.*" + undef_regex, hunk, re.MULTILINE):
+ files.append(filename)
+ break
if not files:
return None
>From df220badf7764a92b44221a391365178524a15b1 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 26 Jun 2026 05:56:47 -0400
Subject: [PATCH 4/6] [X86] Regenerate peephole-reg-sequence.mir test checks
---
llvm/test/CodeGen/X86/peephole-reg-sequence.mir | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/X86/peephole-reg-sequence.mir b/llvm/test/CodeGen/X86/peephole-reg-sequence.mir
index 52b05a0d2b4c2..d62490f6bc074 100644
--- a/llvm/test/CodeGen/X86/peephole-reg-sequence.mir
+++ b/llvm/test/CodeGen/X86/peephole-reg-sequence.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=x86_64-- -mcpu=skylake-avx512 -run-pass=peephole-opt -verify-machineinstrs %s -o - | FileCheck %s
---
@@ -5,8 +6,8 @@ name: regseq_source_class_differs_from_subreg_use
body: |
bb.0:
; CHECK-LABEL: name: regseq_source_class_differs_from_subreg_use
- ; CHECK: [[SRC:%[0-9]+]]:vr128 = AVX512_128_SET0
- ; CHECK-NEXT: [[USE:%[0-9]+]]:vr128 = MOVAPSrr [[SRC]]
+ ; CHECK: [[AVX512_128_SET0_:%[0-9]+]]:vr128 = AVX512_128_SET0
+ ; CHECK-NEXT: [[MOVAPSrr:%[0-9]+]]:vr128 = MOVAPSrr [[AVX512_128_SET0_]]
; CHECK-NEXT: RET 0
%0:vr128x = AVX512_128_SET0
%1:vr256 = REG_SEQUENCE %0, %subreg.sub_xmm
@@ -20,6 +21,9 @@ name: regseq_fold_preserves_source_subreg
body: |
bb.0:
; CHECK-LABEL: name: regseq_fold_preserves_source_subreg
+ ; CHECK: [[AVX_SET0_:%[0-9]+]]:vr256 = AVX_SET0
+ ; CHECK-NEXT: [[MOVAPSrr:%[0-9]+]]:vr128 = MOVAPSrr [[AVX_SET0_]].sub_xmm
+ ; CHECK-NEXT: RET 0
%0:vr256 = AVX_SET0
%1:vr256 = REG_SEQUENCE %0.sub_xmm, %subreg.sub_xmm
%2:vr128 = MOVAPSrr %1.sub_xmm
>From 5f4e7a42caafd6a7a2edb3b432b5331ca04d1a47 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 26 Jun 2026 10:27:08 -0400
Subject: [PATCH 5/6] Revert "[utils] Don't flag undef when replacing existing
uses in same hunk"
This reverts commit 82fcc2cb56858516e2c2d6d3af06ac6a6c20013c.
---
llvm/utils/git/code-format-helper.py | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 47a5b0f06efd9..1e8c332554a8e 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -426,17 +426,9 @@ def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str
undef_regex = r"(?<!%)\bundef\b"
else:
undef_regex = r"UndefValue::get"
-
- # Split the file diff into hunks and search for additions
- # of undef in each hunk.
- hunks = re.split(r"^@@", file, flags=re.MULTILINE)
- for hunk in hunks:
- if re.findall(r"^[+].*" + undef_regex, hunk, re.MULTILINE):
- # If there are removals of undef in this hunk, this is
- # likely a legitimate transformation.
- if not re.findall(r"^-.*" + undef_regex, hunk, re.MULTILINE):
- files.append(filename)
- break
+ # search for additions of undef
+ if re.search(r"^[+].*" + undef_regex, file, re.MULTILINE):
+ files.append(filename)
if not files:
return None
>From 529ee105b64fb650e94a270255d77d8395286d4d Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 26 Jun 2026 10:19:06 -0400
Subject: [PATCH 6/6] [Github] Skip .ll files with autogenerated MIR checks in
undef deprecator
Skip .ll files with MIR check lines that were generated by
update_mir_test_checks.py. Note that .mir files are already
skipped by filter_changed_files.
---
llvm/utils/git/code-format-helper.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 1e8c332554a8e..859a37b490a9b 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -422,6 +422,20 @@ def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str
lines = file.splitlines()
match = re.match("a/([^ ]+)", lines[0] if lines else "")
filename = match[1] if match else ""
+
+ # Skip .ll files with autogenerated MIR checks. MIR should
+ # not be checked in general, but we only skip files
+ # containing the note from the update script since that is
+ # easy to do.
+ if filename.endswith(".ll"):
+ try:
+ with open(filename, "r") as f:
+ first_line = f.readline()
+ if re.search(r"NOTE:.*autogenerated.*update_mir_test_checks", first_line, re.IGNORECASE):
+ continue
+ except (FileNotFoundError, IOError):
+ continue
+
if filename.endswith(".ll"):
undef_regex = r"(?<!%)\bundef\b"
else:
More information about the llvm-commits
mailing list