[llvm-branch-commits] [llvm] [2/3][AMDGPU] Physical register tracking in GCN trackers. (PR #184275)
Dhruva Chakrabarti via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 3 19:29:58 PDT 2026
https://github.com/dhruvachak updated https://github.com/llvm/llvm-project/pull/184275
>From c78451c8109df20fbff4dc4cc5514d7aa76c8a42 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Thu, 2 Jul 2026 14:59:21 -0500
Subject: [PATCH 1/2] [AMDGPU] Add physical register support in GCN trackers
Squashed physical-register tracking work (28 commits) onto the virtual
rename base, to be rebased as a single unit.
Includes:
- Physical register live-in/live-out seeding in GCN RP trackers
- LiveRegUnits-based physical register tracking
- Early-clobber handling for physical registers
- Unified GCNRegPressure objects for virtual and physical
- Tests for physical liveins/liveouts, early clobber, aliasing/allocatable
Assisted-by: Cursor/Claude Opus
---
.../Target/AMDGPU/AMDGPUNextUseAnalysis.cpp | 2 +-
.../Target/AMDGPU/GCNIterativeScheduler.cpp | 18 +-
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 411 +++--
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 147 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 50 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 6 +
.../lib/Target/AMDGPU/SIFormMemoryClauses.cpp | 11 +-
.../machine-scheduler-sink-trivial-remats.mir | 20 +-
.../AMDGPU/materialize-frame-index-sgpr.ll | 1559 +++++++++++++++++
.../regpressure-physreg-early-clobber.mir | 43 +
.../AMDGPU/regpressure-physreg-limits.mir | 239 +++
.../CodeGen/AMDGPU/sched-physreg-liveins.mir | 68 +
.../CodeGen/AMDGPU/sched-physreg-liveouts.mir | 84 +
.../schedule-amdgpu-tracker-physreg-crash.ll | 12 +-
.../AMDGPU/schedule-amdgpu-tracker-physreg.ll | 37 +-
.../AMDGPU/schedule-gcn-physreg-pressure.ll | 630 +++++++
.../Target/AMDGPU/GCNRegPressureTest.cpp | 9 +-
17 files changed, 3170 insertions(+), 176 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
index c5184d4f42b6f..3e3976a7490f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
@@ -2431,7 +2431,7 @@ void printNextUseDistancesAsJson(json::OStream &J, const MachineFunction &MF,
// We don't actually care about register pressure here - just using
// GCNDownwardRPTracker as a convenient way of getting the set of live
// registers at a given instruction.
- GCNDownwardRPTracker RPTracker(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MRI);
ModuleSlotTracker MST(M);
MST.incorporateFunction(F);
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index f2e91e9c1be70..59d9f48faa0c8 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -238,11 +238,8 @@ class SchedStrategyStub : public MachineSchedStrategy {
GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
StrategyKind S)
- : BaseClass(C, std::make_unique<SchedStrategyStub>())
- , Context(C)
- , Strategy(S)
- , UPTracker(*LIS) {
-}
+ : BaseClass(C, std::make_unique<SchedStrategyStub>()), Context(C),
+ Strategy(S), UPTracker(*LIS, Context->MF->getRegInfo()) {}
// returns max pressure for a region
GCNRegPressure
@@ -260,7 +257,7 @@ GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
auto AfterBottomMI = std::next(BottomMI);
if (AfterBottomMI == BBEnd ||
&*AfterBottomMI != UPTracker.getLastTrackedMI()) {
- UPTracker.reset(*BottomMI);
+ UPTracker.reset(*BottomMI, Begin->getParent());
} else {
assert(UPTracker.isValid());
}
@@ -280,16 +277,17 @@ GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
template <typename Range> GCNRegPressure
GCNIterativeScheduler::getSchedulePressure(const Region &R,
Range &&Schedule) const {
- auto const BBEnd = R.Begin->getParent()->end();
- GCNUpwardRPTracker RPTracker(*LIS);
+ const MachineBasicBlock *MBB = R.Begin->getParent();
+ auto const BBEnd = MBB->end();
+ GCNUpwardRPTracker RPTracker(*LIS, MF.getRegInfo());
if (R.End != BBEnd) {
// R.End points to the boundary instruction but the
// schedule doesn't include it
- RPTracker.reset(*R.End);
+ RPTracker.reset(*R.End, MBB);
RPTracker.recede(*R.End);
} else {
// R.End doesn't point to the boundary instruction
- RPTracker.reset(*std::prev(BBEnd));
+ RPTracker.reset(*std::prev(BBEnd), MBB);
}
for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
RPTracker.recede(*getMachineInstr(*--I));
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index a3d198db24e4b..c9053c3ce93be 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -47,10 +47,8 @@ unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,
: (STI->isVectorSuperClass(RC) ? AVGPR : VGPR));
}
-void GCNRegPressure::inc(unsigned Reg,
- LaneBitmask PrevMask,
- LaneBitmask NewMask,
- const MachineRegisterInfo &MRI) {
+void GCNRegPressure::inc(unsigned Reg, LaneBitmask PrevMask,
+ LaneBitmask NewMask, const MachineRegisterInfo &MRI) {
unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(NewMask);
unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(PrevMask);
if (NewNumCoveredRegs == PrevNumCoveredRegs)
@@ -98,6 +96,34 @@ void GCNRegPressure::inc(unsigned Reg,
Value[RegKind] += Sign;
}
+unsigned GCNRegPressure::pressureSetToRegKind(unsigned PSetID) {
+ switch (PSetID) {
+ case AMDGPU::RegisterPressureSets::SReg_32:
+ return SGPR;
+ case AMDGPU::RegisterPressureSets::AGPR_32:
+ return AGPR;
+ case AMDGPU::RegisterPressureSets::VGPR_32:
+ return VGPR;
+ }
+ llvm_unreachable("unexpected pressure set");
+}
+
+// Adjusts both raw count and tuple weight per unit. Raw count and
+// tuple weight receive identical increments. This means 32-bit physical
+// registers contribute to tuple weight (unlike virtual registers where only
+// tuples > 32-bit contribute).
+void GCNRegPressure::adjustPhysUnitPressure(MCRegUnit Unit, bool IsAdd,
+ const SIRegisterInfo &SRI) {
+ const int *PSetIDs = SRI.getRegUnitPressureSets(Unit);
+ if (PSetIDs[0] == -1)
+ return;
+ assert(PSetIDs[1] == -1 && "expected single pressure set per unit");
+ unsigned Kind = pressureSetToRegKind(PSetIDs[0]);
+ int Delta = (IsAdd ? 1 : -1) * static_cast<int>(SRI.getRegUnitWeight(Unit));
+ Value[Kind] += Delta;
+ Value[TOTAL_KINDS + Kind] += Delta;
+}
+
namespace {
struct RegExcess {
unsigned SGPR = 0;
@@ -485,6 +511,60 @@ LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI, LaneMaskFilter);
}
+bool GCNRPTracker::isUnitLiveAt(MCRegUnit Unit, SlotIndex SI) const {
+ const LiveRange *LR = LIS.getCachedRegUnit(Unit);
+ // If LIS has no reg-unit live range, be conservative and assume it is live.
+ return !LR || LR->liveAt(SI);
+}
+
+void GCNRPTracker::addUnitsAndIncPressure(MCRegister Reg,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (!PhysLiveRegUnits.test(U)) {
+ PhysLiveRegUnits.set(U);
+ Pressure.inc(Unit, *SRI);
+ }
+ }
+}
+
+void GCNRPTracker::removeUnitsAndDecPressure(MCRegister Reg,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U)) {
+ PhysLiveRegUnits.reset(U);
+ Pressure.dec(Unit, *SRI);
+ }
+ }
+}
+
+void GCNRPTracker::removeKilledUnitsAndDecPressure(MCRegister Reg, SlotIndex SI,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SI)) {
+ PhysLiveRegUnits.reset(U);
+ Pressure.dec(Unit, *SRI);
+ }
+ }
+}
+
+void llvm::addPhysRegPressure(GCNRegPressure &RP, const SIRegisterInfo &SRI,
+ const BitVector &PhysLiveUnits) {
+ for (unsigned U : PhysLiveUnits.set_bits())
+ RP.inc(static_cast<MCRegUnit>(U), SRI);
+}
+
+GCNRegPressure GCNRPTracker::constructPhysRegPressure() const {
+ GCNRegPressure Res;
+ addPhysRegPressure(Res, *SRI, PhysLiveRegUnits);
+ return Res;
+}
+
LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
const MachineRegisterInfo &MRI,
LaneBitmask LaneMaskFilter) {
@@ -557,18 +637,46 @@ void GCNRPTracker::reset(const MachineBasicBlock &MBB, bool End) {
void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
this->MRI = &MRI;
+ SRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
LastTrackedMI = nullptr;
VirtLiveRegs = llvm::getVirtLiveRegs(SI, LIS, MRI);
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
+
+ updatePhysRegTracking();
+ // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
+ // stale data if physical tracking was previously enabled.
+ PhysLiveRegUnits.reset();
}
void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
const LiveRegSet &VirtLiveRegs) {
this->MRI = &MRI;
+ SRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
LastTrackedMI = nullptr;
if (&this->VirtLiveRegs != &VirtLiveRegs)
this->VirtLiveRegs = VirtLiveRegs;
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
+
+ updatePhysRegTracking();
+ // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
+ // stale data if physical tracking was previously enabled.
+ PhysLiveRegUnits.reset();
+}
+
+void GCNRPTracker::reset(const MachineRegisterInfo &MRInfo,
+ const LiveRegSet &VirtLiveRegsSet,
+ const BitVector &PhysLiveUnits) {
+ reset(MRInfo, VirtLiveRegsSet);
+ initPhysLiveUnits(PhysLiveUnits);
+}
+
+void GCNRPTracker::initPhysLiveUnits(const BitVector &PhysLiveUnits) {
+ if (!TrackPhysRegs)
+ return;
+ PhysLiveRegUnits = PhysLiveUnits;
+ GCNRegPressure PhysPressure = constructPhysRegPressure();
+ CurPressure += PhysPressure;
+ MaxPressure = max(MaxPressure, CurPressure);
}
/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
@@ -596,29 +704,37 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
GCNRegPressure DefPressure, ECDefPressure;
bool HasECDefs = false;
for (const MachineOperand &MO : MI.all_defs()) {
- if (!MO.getReg().isVirtual())
- continue;
-
Register Reg = MO.getReg();
- LaneBitmask DefMask = getDefRegMask(MO, *MRI);
- // Treat a def as fully live at the moment of definition: keep a record.
- if (MO.isEarlyClobber()) {
- ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
- HasECDefs = true;
- } else
- DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+ if (Reg.isVirtual()) {
+ LaneBitmask DefMask = getDefRegMask(MO, *MRI);
- auto I = VirtLiveRegs.find(Reg);
- if (I == VirtLiveRegs.end())
- continue;
+ // Treat a def as fully live at the moment of definition: keep a record.
+ if (MO.isEarlyClobber()) {
+ ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+ HasECDefs = true;
+ } else
+ DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+
+ auto I = VirtLiveRegs.find(Reg);
+ if (I == VirtLiveRegs.end())
+ continue;
+
+ LaneBitmask &LiveMask = I->second;
+ LaneBitmask PrevMask = LiveMask;
+ LiveMask &= ~DefMask;
+ CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ if (LiveMask.none())
+ VirtLiveRegs.erase(I);
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (MO.isEarlyClobber()) {
+ for (MCRegUnit Unit : SRI->regunits(Reg.asMCReg()))
+ ECDefPressure.inc(Unit, *SRI);
+ HasECDefs = true;
+ }
- LaneBitmask &LiveMask = I->second;
- LaneBitmask PrevMask = LiveMask;
- LiveMask &= ~DefMask;
- CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
- if (LiveMask.none())
- VirtLiveRegs.erase(I);
+ removeUnitsAndDecPressure(Reg.asMCReg(), CurPressure);
+ }
}
// Update MaxPressure with defs pressure.
@@ -637,11 +753,31 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
CurPressure.inc(U.VRegOrUnit.asVirtualReg(), PrevMask, LiveMask, *MRI);
}
+ if (TrackPhysRegs) {
+ for (const MachineOperand &MO : MI.all_uses()) {
+ if (!MO.readsReg())
+ continue;
+ Register Reg = MO.getReg();
+ if (!shouldTrackPhysReg(Reg))
+ continue;
+ addUnitsAndIncPressure(Reg.asMCReg(), CurPressure);
+ }
+ }
+
// Update MaxPressure with uses plus early-clobber defs pressure.
MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)
: max(CurPressure, MaxPressure);
- assert(CurPressure == getVirtRegPressure(*MRI, VirtLiveRegs));
+#ifndef NDEBUG
+ auto VirtPressure = getVirtRegPressure(*MRI, VirtLiveRegs);
+ auto PhysPressure = constructPhysRegPressure();
+ assert(CurPressure == VirtPressure + PhysPressure ||
+ (dbgs() << "Pressure mismatch in recede()\nMI: " << MI
+ << "Tracked: " << print(CurPressure) << "Expected: "
+ << print(VirtPressure + PhysPressure) << "Virt: "
+ << print(VirtPressure) << "Phys: " << print(PhysPressure),
+ false));
+#endif
}
////////////////////////////////////////////////////////////////////////////////
@@ -649,7 +785,8 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy) {
+ const LiveRegSet *VirtLiveRegsCopy,
+ const MachineBasicBlock *SeedPhysMBB) {
MBBEnd = MI.getParent()->end();
assert(End == MBBEnd ||
End->getParent()->end() == MBBEnd && "end unrelated to MI block");
@@ -666,9 +803,22 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
GCNRPTracker::reset(*NextMI, /*After=*/false);
else
GCNRPTracker::reset(*MI.getParent(), /*End=*/true);
+
+ if (SeedPhysMBB && TrackPhysRegs &&
+ MI.getMF()->getProperties().hasTracksLiveness())
+ initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveins());
+
return NextMI != End;
}
+bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
+ const LiveRegSet &VirtLiveRegs,
+ const BitVector &PhysLiveUnits) {
+ bool Result = reset(MI, MI.getParent()->end(), &VirtLiveRegs);
+ initPhysLiveUnits(PhysLiveUnits);
+ return Result;
+}
+
bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
bool UseInternalIterator) {
assert(MRI && "call reset first");
@@ -693,38 +843,46 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
// Remove dead registers or mask bits.
SmallSet<Register, 8> SeenRegs;
- for (auto &MO : CurrMI->operands()) {
- if (!MO.isReg() || !MO.getReg().isVirtual())
- continue;
- if (MO.isUse() && !MO.readsReg())
- continue;
- if (!UseInternalIterator && MO.isDef())
- continue;
- if (!SeenRegs.insert(MO.getReg()).second)
+ for (const auto &MO : CurrMI->operands()) {
+ if (!MO.isReg())
continue;
- const LiveInterval &LI = LIS.getInterval(MO.getReg());
- if (LI.hasSubRanges()) {
- auto It = VirtLiveRegs.end();
- for (const auto &S : LI.subranges()) {
- if (!S.liveAt(SI)) {
- if (It == VirtLiveRegs.end()) {
- It = VirtLiveRegs.find(MO.getReg());
- if (It == VirtLiveRegs.end())
- llvm_unreachable("register isn't live");
+ Register Reg = MO.getReg();
+
+ if (Reg.isVirtual()) {
+ if (MO.isUse() && !MO.readsReg())
+ continue;
+ if (!UseInternalIterator && MO.isDef())
+ continue;
+ if (!SeenRegs.insert(Reg).second)
+ continue;
+ const LiveInterval &LI = LIS.getInterval(Reg);
+ if (LI.hasSubRanges()) {
+ auto It = VirtLiveRegs.end();
+ for (const auto &S : LI.subranges()) {
+ if (!S.liveAt(SI)) {
+ if (It == VirtLiveRegs.end()) {
+ It = VirtLiveRegs.find(Reg);
+ if (It == VirtLiveRegs.end())
+ llvm_unreachable("register isn't live");
+ }
+ auto PrevMask = It->second;
+ It->second &= ~S.LaneMask;
+ CurPressure.inc(Reg, PrevMask, It->second, *MRI);
}
- auto PrevMask = It->second;
- It->second &= ~S.LaneMask;
- CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);
}
- }
- if (It != VirtLiveRegs.end() && It->second.none())
+ if (It != VirtLiveRegs.end() && It->second.none())
+ VirtLiveRegs.erase(It);
+ } else if (!LI.liveAt(SI)) {
+ auto It = VirtLiveRegs.find(Reg);
+ if (It == VirtLiveRegs.end())
+ llvm_unreachable("register isn't live");
+ CurPressure.inc(Reg, It->second, LaneBitmask::getNone(), *MRI);
VirtLiveRegs.erase(It);
- } else if (!LI.liveAt(SI)) {
- auto It = VirtLiveRegs.find(MO.getReg());
- if (It == VirtLiveRegs.end())
- llvm_unreachable("register isn't live");
- CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);
- VirtLiveRegs.erase(It);
+ }
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (!SeenRegs.insert(Reg).second)
+ continue;
+ removeKilledUnitsAndDecPressure(Reg.asMCReg(), SI, CurPressure);
}
}
@@ -749,12 +907,16 @@ void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,
// Add new registers or mask bits.
for (const auto &MO : CurrMI->all_defs()) {
Register Reg = MO.getReg();
- if (!Reg.isVirtual())
- continue;
- auto &LiveMask = VirtLiveRegs[Reg];
- auto PrevMask = LiveMask;
- LiveMask |= getDefRegMask(MO, *MRI);
- CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+
+ if (Reg.isVirtual()) {
+ auto &LiveMask = VirtLiveRegs[Reg];
+ auto PrevMask = LiveMask;
+ LiveMask |= getDefRegMask(MO, *MRI);
+ CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (!MO.isDead())
+ addUnitsAndIncPressure(Reg.asMCReg(), CurPressure);
+ }
}
MaxPressure = max(MaxPressure, CurPressure);
@@ -782,10 +944,13 @@ bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy) {
- if (!reset(*Begin, End, VirtLiveRegsCopy))
- return false;
- return advance(End);
+ const LiveRegSet *VirtLiveRegsCopy,
+ const BitVector *PhysLiveUnits) {
+ bool ResetStatus = reset(*Begin, End, VirtLiveRegsCopy);
+ // Restore the physical snapshot regardless of the above status.
+ if (PhysLiveUnits)
+ initPhysLiveUnits(*PhysLiveUnits);
+ return ResetStatus && advance(End);
}
Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
@@ -827,50 +992,63 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
GCNRegPressure TempPressure = CurPressure;
+ // Process uses: decrement pressure for last-use lanes (virtual) or
+ // killed units (physical).
for (const VRegMaskOrUnit &Use : RegOpers.Uses) {
- if (!Use.VRegOrUnit.isVirtualReg())
- continue;
- Register Reg = Use.VRegOrUnit.asVirtualReg();
- LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
- if (LastUseMask.none())
- continue;
- // The LastUseMask is queried from the liveness information of instruction
- // which may be further down the schedule. Some lanes may actually not be
- // last uses for the current position.
- // FIXME: allow the caller to pass in the list of vreg uses that remain
- // to be bottom-scheduled to avoid searching uses at each query.
- SlotIndex CurrIdx;
- const MachineBasicBlock *MBB = MI->getParent();
- MachineBasicBlock::const_iterator IdxPos = skipDebugInstructionsForward(
- LastTrackedMI ? LastTrackedMI : MBB->begin(), MBB->end());
- if (IdxPos == MBB->end()) {
- CurrIdx = LIS.getMBBEndIdx(MBB);
- } else {
- CurrIdx = LIS.getInstructionIndex(*IdxPos).getRegSlot();
- }
+ if (Use.VRegOrUnit.isVirtualReg()) {
+ Register Reg = Use.VRegOrUnit.asVirtualReg();
+ LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
+ if (LastUseMask.none())
+ continue;
+ // The LastUseMask is queried from the liveness information of instruction
+ // which may be further down the schedule. Some lanes may actually not be
+ // last uses for the current position.
+ // FIXME: allow the caller to pass in the list of vreg uses that remain
+ // to be bottom-scheduled to avoid searching uses at each query.
+ SlotIndex CurrIdx;
+ const MachineBasicBlock *MBB = MI->getParent();
+ MachineBasicBlock::const_iterator IdxPos = skipDebugInstructionsForward(
+ LastTrackedMI ? LastTrackedMI : MBB->begin(), MBB->end());
+ if (IdxPos == MBB->end()) {
+ CurrIdx = LIS.getMBBEndIdx(MBB);
+ } else {
+ CurrIdx = LIS.getInstructionIndex(*IdxPos).getRegSlot();
+ }
- LastUseMask =
- findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);
- if (LastUseMask.none())
- continue;
+ LastUseMask =
+ findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);
+ if (LastUseMask.none())
+ continue;
- auto It = VirtLiveRegs.find(Reg);
- LaneBitmask LiveMask =
- It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
- LaneBitmask NewMask = LiveMask & ~LastUseMask;
- TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ auto It = VirtLiveRegs.find(Reg);
+ LaneBitmask LiveMask =
+ It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
+ LaneBitmask NewMask = LiveMask & ~LastUseMask;
+ TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ } else if (TrackPhysRegs) {
+ MCRegUnit Unit = Use.VRegOrUnit.asMCRegUnit();
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SlotIdx))
+ TempPressure.dec(Unit, *SRI);
+ }
}
- // Generate liveness for defs.
+ // Process defs: increment pressure for new lanes (virtual) or
+ // new units (physical).
for (const VRegMaskOrUnit &Def : RegOpers.Defs) {
- if (!Def.VRegOrUnit.isVirtualReg())
- continue;
- Register Reg = Def.VRegOrUnit.asVirtualReg();
- auto It = VirtLiveRegs.find(Reg);
- LaneBitmask LiveMask =
- It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
- LaneBitmask NewMask = LiveMask | Def.LaneMask;
- TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ if (Def.VRegOrUnit.isVirtualReg()) {
+ Register Reg = Def.VRegOrUnit.asVirtualReg();
+ auto It = VirtLiveRegs.find(Reg);
+ LaneBitmask LiveMask =
+ It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
+ LaneBitmask NewMask = LiveMask | Def.LaneMask;
+ TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ } else if (TrackPhysRegs) {
+ MCRegUnit Unit = Def.VRegOrUnit.asMCRegUnit();
+ unsigned U = static_cast<unsigned>(Unit);
+ if (!PhysLiveRegUnits.test(U))
+ TempPressure.inc(Unit, *SRI);
+ }
}
return TempPressure;
@@ -885,14 +1063,15 @@ bool GCNUpwardRPTracker::isValid() const {
dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
" LIS reported livesets mismatch:\n"
<< print(LISLR, *MRI);
- reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());
+ reportMismatch(LISLR, TrackedLR, SRI);
return false;
}
- auto LISPressure = getVirtRegPressure(*MRI, LISLR);
- if (LISPressure != CurPressure) {
+ auto ComputedPressure =
+ getVirtRegPressure(*MRI, LISLR) + constructPhysRegPressure();
+ if (ComputedPressure != CurPressure) {
dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
- << print(CurPressure) << "LIS rpt: " << print(LISPressure);
+ << print(CurPressure) << "Computed rpt: " << print(ComputedPressure);
return false;
}
return true;
@@ -1000,9 +1179,18 @@ bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
if (MBB.empty()) {
VirtLiveIn = VirtLiveOut = getVirtLiveRegs(MBBStartSlot, LIS, MRI);
RPAtMBBEnd = getVirtRegPressure(MRI, VirtLiveIn);
+ const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
+ BitVector SeenUnits(SRI->getNumRegUnits());
+ for (const auto &LI : MBB.liveins())
+ if (MRI.isAllocatable(LI.PhysReg))
+ for (MCRegUnit Unit : SRI->regunits(LI.PhysReg))
+ if (!SeenUnits.test(static_cast<unsigned>(Unit))) {
+ SeenUnits.set(static_cast<unsigned>(Unit));
+ RPAtMBBEnd.inc(Unit, *SRI);
+ }
} else {
- GCNDownwardRPTracker RPT(LIS);
- RPT.reset(MBB.front(), MBB.end());
+ GCNDownwardRPTracker RPT(LIS, MRI);
+ RPT.reset(MBB.front(), MBB.end(), /*VirtLiveRegs=*/nullptr, &MBB);
VirtLiveIn = RPT.getVirtLiveRegs();
@@ -1016,8 +1204,8 @@ bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
RPAtMBBEnd = RPT.getPressure();
}
} else {
- GCNUpwardRPTracker RPT(LIS);
- RPT.reset(MRI, MBBLastSlot);
+ GCNUpwardRPTracker RPT(LIS, MRI);
+ RPT.reset(MRI, MBBLastSlot, &MBB);
VirtLiveOut = RPT.getVirtLiveRegs();
RPAtMBBEnd = RPT.getPressure();
@@ -1088,9 +1276,10 @@ LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,
unsigned MaxNumRegs = 0;
const MachineInstr *MaxPressureMI = nullptr;
- GCNUpwardRPTracker RPT(LIS);
+ GCNUpwardRPTracker RPT(LIS, MRI);
for (const MachineBasicBlock &MBB : MF) {
- RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
+ RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot(),
+ &MBB);
for (const MachineInstr &MI : reverse(MBB)) {
RPT.recede(MI);
unsigned NumRegs = RPT.getMaxPressure().getNumRegs(Kind);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 1c09042a5784c..92819b15eaaf2 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -18,6 +18,7 @@
#define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
#include "GCNSubtarget.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/RegisterPressure.h"
#include <algorithm>
@@ -125,11 +126,28 @@ struct GCNRegPressure {
return std::max(UnifiedSpill, ArchSpill + AGPRSpill);
}
+ /// Adjust pressure for a virtual register.
void inc(unsigned Reg,
LaneBitmask PrevMask,
LaneBitmask NewMask,
const MachineRegisterInfo &MRI);
+ /// Increment pressure for a physical register unit.
+ void inc(MCRegUnit Unit, const SIRegisterInfo &SRI) {
+ adjustPhysUnitPressure(Unit, /*IsAdd=*/true, SRI);
+ }
+
+ /// Decrement pressure for a physical register unit.
+ void dec(MCRegUnit Unit, const SIRegisterInfo &SRI) {
+ adjustPhysUnitPressure(Unit, /*IsAdd=*/false, SRI);
+ }
+
+private:
+ static unsigned pressureSetToRegKind(unsigned PSetID);
+ void adjustPhysUnitPressure(MCRegUnit Unit, bool IsAdd,
+ const SIRegisterInfo &SRI);
+
+public:
bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure &O,
unsigned DynamicVGPRBlockSize) const {
return getOccupancy(ST, DynamicVGPRBlockSize) >
@@ -324,12 +342,30 @@ class GCNRPTracker {
protected:
const LiveIntervals &LIS;
+ mutable const MachineRegisterInfo *MRI = nullptr;
+ const SIRegisterInfo *SRI = nullptr;
+
LiveRegSet VirtLiveRegs;
+
+ // Physical register liveness tracked at the register-unit level.
+ // Each bit corresponds to a register unit. This avoids aliasing issues
+ // since overlapping physical registers share the same underlying units.
+ BitVector PhysLiveRegUnits;
+
GCNRegPressure CurPressure, MaxPressure;
+
+ // Flag to control whether physical register tracking is active.
+ // Set to true when GCNTrackers are enabled, false otherwise.
+ bool TrackPhysRegs = false;
+
const MachineInstr *LastTrackedMI = nullptr;
- mutable const MachineRegisterInfo *MRI = nullptr;
- GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
+ GCNRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : LIS(LIS), MRI(&MRI),
+ SRI(static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo())),
+ PhysLiveRegUnits(SRI->getNumRegUnits()) {
+ updatePhysRegTracking();
+ }
/// Resets tracker before or \p After the provided \p MI, which can be a debug
/// instruction.
@@ -346,14 +382,60 @@ class GCNRPTracker {
LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;
+ bool shouldTrackPhysReg(Register Reg) const {
+ return TrackPhysRegs && Reg.isPhysical() && MRI->isAllocatable(Reg);
+ }
+
+ // Check if a register unit is live at a given slot index per LIS.
+ bool isUnitLiveAt(MCRegUnit Unit, SlotIndex SI) const;
+
+ // Construct physical register pressure from PhysLiveRegUnits.
+ GCNRegPressure constructPhysRegPressure() const;
+
+ // Add units of Reg that are not already live. Increases Pressure for each
+ // newly live unit.
+ void addUnitsAndIncPressure(MCRegister Reg, GCNRegPressure &Pressure);
+
+ // Remove all live units of Reg. Decreases Pressure for each removed unit.
+ void removeUnitsAndDecPressure(MCRegister Reg, GCNRegPressure &Pressure);
+
+ // Remove units of Reg that are currently live but killed at SI.
+ // Decreases Pressure for each killed unit.
+ void removeKilledUnitsAndDecPressure(MCRegister Reg, SlotIndex SI,
+ GCNRegPressure &Pressure);
+
public:
+ // Enable physical register tracking only if both GCNTrackers and
+ // TrackPhysRegInTrackers are true.
+ void updatePhysRegTracking();
+
/// Resets tracker with the provided \p VirtLiveRegs.
void reset(const MachineRegisterInfo &MRI, const LiveRegSet &VirtLiveRegs);
+ // Reset tracker with both virtual and physical live register state.
+ void reset(const MachineRegisterInfo &MRInfo,
+ const LiveRegSet &VirtLiveRegsSet, const BitVector &PhysLiveUnits);
+
// live regs for the current state
const decltype(VirtLiveRegs) &getVirtLiveRegs() const { return VirtLiveRegs; }
+ const BitVector &getPhysLiveRegUnits() const { return PhysLiveRegUnits; }
const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
+ /// Initialize PhysLiveRegUnits from a range of RegisterMaskPair entries
+ /// and update CurPressure/MaxPressure accordingly.
+ template <typename RangeT>
+ void initPhysLiveUnitsFromRegMaskPairs(RangeT &&Pairs) {
+ assert(TrackPhysRegs && "physical register tracking must be enabled");
+ for (const auto &RM : Pairs)
+ if (MRI->isAllocatable(RM.PhysReg))
+ addUnitsAndIncPressure(RM.PhysReg, CurPressure);
+ MaxPressure = max(MaxPressure, CurPressure);
+ }
+
+ /// Restore PhysLiveRegUnits from a previously saved BitVector and update
+ /// CurPressure/MaxPressure accordingly.
+ void initPhysLiveUnits(const BitVector &PhysLiveUnits);
+
void clearMaxPressure() { MaxPressure.clear(); }
GCNRegPressure getPressure() const { return CurPressure; }
@@ -371,13 +453,36 @@ getVirtLiveRegs(SlotIndex SI, const LiveIntervals &LIS,
class GCNUpwardRPTracker : public GCNRPTracker {
public:
- GCNUpwardRPTracker(const LiveIntervals &LIS) : GCNRPTracker(LIS) {}
+ GCNUpwardRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : GCNRPTracker(LIS, MRI) {}
using GCNRPTracker::reset;
- /// Resets tracker to the point just after \p MI (in program order), which can
- /// be a debug instruction.
- void reset(const MachineInstr &MI) { reset(MI, /*After=*/true); }
+ /// reset tracker at the specified slot index \p SI. If \p SeedPhysMBB is
+ /// non-null, also seed physical live-out state from that MBB's successors.
+ void reset(const MachineRegisterInfo &MRI, SlotIndex SI,
+ const MachineBasicBlock *SeedPhysMBB = nullptr) {
+ GCNRPTracker::reset(MRI, llvm::getVirtLiveRegs(SI, LIS, MRI));
+ if (SeedPhysMBB && TrackPhysRegs &&
+ SeedPhysMBB->getParent()->getProperties().hasTracksLiveness())
+ initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveouts());
+ }
+
+ /// reset tracker to the end of the \p MBB and seed physical live-outs
+ /// from the MBB's successors.
+ void reset(const MachineBasicBlock &MBB) {
+ SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(&MBB);
+ reset(MBB.getParent()->getRegInfo(), MBBLastSlot, &MBB);
+ }
+
+ /// reset tracker to the point just after \p MI (in program order).
+ /// If \p SeedPhysMBB is non-null, also seed physical live-out state from
+ /// that MBB's successors.
+ void reset(const MachineInstr &MI,
+ const MachineBasicBlock *SeedPhysMBB = nullptr) {
+ reset(MI.getMF()->getRegInfo(), LIS.getInstructionIndex(MI).getDeadSlot(),
+ SeedPhysMBB);
+ }
/// Move to the state of RP just before the \p MI . If \p UseInternalIterator
/// is set, also update the internal iterators. Setting \p UseInternalIterator
@@ -409,7 +514,8 @@ class GCNDownwardRPTracker : public GCNRPTracker {
MachineBasicBlock::const_iterator MBBEnd;
public:
- GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
+ GCNDownwardRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : GCNRPTracker(LIS, MRI) {}
using GCNRPTracker::reset;
@@ -424,10 +530,18 @@ class GCNDownwardRPTracker : public GCNRPTracker {
/// Reset tracker to the point before the \p MI filling \p VirtLiveRegs upon
/// this point using LIS. \p End must be between the MI and the end of its
- /// parent block (inclusive). \p returns false if the range [MI, End) is empty
- /// except debug values.
+ /// parent block (inclusive). If \p SeedPhysMBB is non-null, also seed
+ /// physical live-in state from that MBB's live-in list. \p returns false if
+ /// the range [MI, End) is empty except debug values.
bool reset(const MachineInstr &MI, MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegs = nullptr);
+ const LiveRegSet *VirtLiveRegs = nullptr,
+ const MachineBasicBlock *SeedPhysMBB = nullptr);
+
+ /// Reset tracker to the point before \p MI, restoring both virtual and
+ /// physical register state from saved snapshots.
+ /// \p returns false if block is empty except debug values.
+ bool reset(const MachineInstr &MI, const LiveRegSet &VirtLiveRegs,
+ const BitVector &PhysLiveUnits);
/// Move to the state right before the next MI or after the end of MBB.
/// \p returns false if reached end of the block.
@@ -465,11 +579,13 @@ class GCNDownwardRPTracker : public GCNRPTracker {
/// Reset tracker to \p Begin (filling \p LiveRegs upon this point using LIS)
/// and advance to \p End, which must be between \p Begin and the end of its
- /// parent block (inclusive). \p returns false if the range [Begin, End) is
- /// empty except debug values.
+ /// parent block (inclusive). If \p PhysLiveUnits is non-null, the physical
+ /// register state is restored from that snapshot after the reset. \p returns
+ /// false if the range [Begin, End) is empty except debug values.
bool advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy = nullptr);
+ const LiveRegSet *VirtLiveRegsCopy = nullptr,
+ const BitVector *PhysLiveUnits = nullptr);
/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
/// Calculate the impact \p MI will have on CurPressure and \return the
@@ -557,6 +673,11 @@ GCNRegPressure getVirtRegPressure(const MachineRegisterInfo &MRI,
return Res;
}
+/// Adds to \p RP the pressure contributed by the live physical register units
+/// set in \p PhysLiveUnits.
+void addPhysRegPressure(GCNRegPressure &RP, const SIRegisterInfo &SRI,
+ const BitVector &PhysLiveUnits);
+
bool isEqual(const GCNRPTracker::LiveRegSet &S1,
const GCNRPTracker::LiveRegSet &S2);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 9b25c8e7cc739..5e30603dc9383 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -78,6 +78,12 @@ static cl::opt<bool> GCNTrackers(
cl::desc("Use the AMDGPU specific RPTrackers during scheduling"),
cl::init(false));
+static cl::opt<bool> TrackPhysRegInTrackers(
+ "amdgpu-trackers-physical-register-tracking", cl::Hidden,
+ cl::desc("When using GCN trackers, count physical registers (e.g. from "
+ "inline asm) in pressure."),
+ cl::init(true));
+
static cl::opt<unsigned> PendingQueueLimit(
"amdgpu-scheduler-pending-queue-limit", cl::Hidden,
cl::desc(
@@ -134,7 +140,8 @@ const unsigned ScheduleMetrics::ScaleFactor = 100;
GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
: GenericScheduler(C), TargetOccupancy(0), MF(nullptr),
- DownwardTracker(*C->LIS), UpwardTracker(*C->LIS), HasHighPressure(false) {
+ DownwardTracker(*C->LIS, C->MF->getRegInfo()),
+ UpwardTracker(*C->LIS, C->MF->getRegInfo()), HasHighPressure(false) {
if (GCNTrackers.getNumOccurrences() > 0)
GCNTrackersOverride = GCNTrackers;
}
@@ -143,7 +150,6 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
GenericScheduler::initialize(DAG);
MF = &DAG->MF;
-
const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
SGPRExcessLimit =
@@ -208,6 +214,14 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
<< ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
}
+void GCNRPTracker::updatePhysRegTracking() {
+ if (!GCNTrackers || !TrackPhysRegInTrackers) {
+ TrackPhysRegs = false;
+ return;
+ }
+ TrackPhysRegs = true;
+}
+
/// Checks whether \p SU can use the cached DAG pressure diffs to compute the
/// current register pressure.
///
@@ -1068,11 +1082,19 @@ void GCNScheduleDAGMILive::schedule() {
GCNRegPressure
GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const {
- if (Regions[RegionIdx].first == Regions[RegionIdx].second)
- return llvm::getVirtRegPressure(MRI, VirtLiveIns[RegionIdx]);
- GCNDownwardRPTracker RPTracker(*LIS);
+ if (Regions[RegionIdx].first == Regions[RegionIdx].second) {
+ GCNRegPressure RP = llvm::getVirtRegPressure(MRI, VirtLiveIns[RegionIdx]);
+ // Fold in physical live-in pressure so that empty regions are consistent
+ // with non-empty regions. This is a no-op when physical tracking is off,
+ // since the snapshot has no set units.
+ const auto *SRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ llvm::addPhysRegPressure(RP, *SRI, PhysLiveIns[RegionIdx]);
+ return RP;
+ }
+ GCNDownwardRPTracker RPTracker(*LIS, MF.getRegInfo());
RPTracker.advance(Regions[RegionIdx].first, Regions[RegionIdx].second,
- &VirtLiveIns[RegionIdx]);
+ &VirtLiveIns[RegionIdx], &PhysLiveIns[RegionIdx]);
return RPTracker.moveMaxPressure();
}
@@ -1084,7 +1106,7 @@ static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin,
void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
const MachineBasicBlock *MBB) {
- GCNDownwardRPTracker RPTracker(*LIS);
+ GCNDownwardRPTracker RPTracker(*LIS, MF.getRegInfo());
// If the block has the only successor then live-ins of that successor are
// live-outs of the current block. We can reuse calculated live set if the
@@ -1117,7 +1139,7 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second);
if (VirtLiveInIt != MBBVirtLiveIns.end()) {
auto VirtLiveIn = std::move(VirtLiveInIt->second);
- RPTracker.reset(*MBB->begin(), MBB->end(), &VirtLiveIn);
+ RPTracker.reset(*MBB->begin(), MBB->end(), &VirtLiveIn, MBB);
MBBVirtLiveIns.erase(VirtLiveInIt);
} else {
I = Rgn.first;
@@ -1125,7 +1147,7 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
#ifdef EXPENSIVE_CHECKS
assert(isEqual(getVirtLiveRegsBefore(*NonDbgMI, *LIS), VirtLiveInSet));
#endif
- RPTracker.reset(*I, I->getParent()->end(), &VirtLiveInSet);
+ RPTracker.reset(*I, I->getParent()->end(), &VirtLiveInSet, MBB);
}
for (;;) {
@@ -1133,10 +1155,12 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
if (Regions[CurRegion].first == I || NonDbgMI == I) {
VirtLiveIns[CurRegion] = RPTracker.getVirtLiveRegs();
+ PhysLiveIns[CurRegion] = RPTracker.getPhysLiveRegUnits();
RPTracker.clearMaxPressure();
}
if (Regions[CurRegion].second == I) {
+ PhysLiveOuts[CurRegion] = RPTracker.getPhysLiveRegUnits();
Pressure[CurRegion] = RPTracker.moveMaxPressure();
if (CurRegion-- == RegionIdx)
break;
@@ -1204,6 +1228,8 @@ void GCNScheduleDAGMILive::finalizeSchedule() {
// MachineScheduler after all regions have been recorded by
// GCNScheduleDAGMILive::schedule().
VirtLiveIns.resize(Regions.size());
+ PhysLiveIns.resize(Regions.size());
+ PhysLiveOuts.resize(Regions.size());
Pressure.resize(Regions.size());
RegionsWithHighRP.resize(Regions.size());
RegionsWithExcessRP.resize(Regions.size());
@@ -1250,9 +1276,11 @@ void GCNScheduleDAGMILive::runSchedStages() {
if (S.useGCNTrackers()) {
const unsigned RegionIdx = Stage->getRegionIdx();
- S.getDownwardTracker()->reset(MRI, VirtLiveIns[RegionIdx]);
+ S.getDownwardTracker()->reset(MRI, VirtLiveIns[RegionIdx],
+ PhysLiveIns[RegionIdx]);
S.getUpwardTracker()->reset(
- MRI, RegionVirtLiveOuts.getVirtLiveRegsForRegionIdx(RegionIdx));
+ MRI, RegionVirtLiveOuts.getVirtLiveRegsForRegionIdx(RegionIdx),
+ PhysLiveOuts[RegionIdx]);
}
ScheduleDAGMILive::schedule();
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index be459ed36d6d8..19dee8b870a82 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -293,6 +293,12 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
// Region live-in cache.
SmallVector<GCNRPTracker::LiveRegSet, 32> VirtLiveIns;
+ // Per-region physical register live-in cache (register unit BitVectors).
+ SmallVector<BitVector, 32> PhysLiveIns;
+
+ // Per-region physical register live-out cache (register unit BitVectors).
+ SmallVector<BitVector, 32> PhysLiveOuts;
+
// Region pressure cache.
SmallVector<GCNRegPressure, 32> Pressure;
diff --git a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
index 45f0e6a8ba1e7..f121fc1b5f206 100644
--- a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
@@ -275,7 +275,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
"amdgpu-max-memory-clause", MaxClause);
for (MachineBasicBlock &MBB : MF) {
- GCNDownwardRPTracker RPT(*LIS);
+ GCNDownwardRPTracker RPT(*LIS, *MRI);
MachineBasicBlock::instr_iterator Next;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
MachineInstr &MI = *I;
@@ -290,16 +290,17 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
continue;
if (!RPT.getNext().isValid())
- RPT.reset(MI, MBB.end());
+ RPT.reset(MI, MBB.end(), /*VirtLiveRegs=*/nullptr, &MBB);
else { // Advance the state to the current MI.
RPT.advance(MachineBasicBlock::const_iterator(MI));
RPT.advanceBeforeNext();
}
const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getVirtLiveRegs());
+ const BitVector PhysRegsCopy(RPT.getPhysLiveRegUnits());
RegUse Defs, Uses;
if (!processRegUses(MI, Defs, Uses, RPT)) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
continue;
}
@@ -323,7 +324,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
++Length;
}
if (Length < 2) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
continue;
}
@@ -391,7 +392,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
}
// Restore the state after processing the end of the bundle.
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
if (!Kill)
continue;
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index b421f07b05877..b6157fb8c0a95 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -11144,8 +11144,6 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-GCNTRACKERS-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
; GFX908-GCNTRACKERS-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -11205,6 +11203,8 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr8, implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr9, implicit $exec, implicit $mode
@@ -11445,8 +11445,6 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-GCNTRACKERS-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
; GFX908-GCNTRACKERS-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -11508,18 +11506,20 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: S_BRANCH %bb.1
; GFX908-GCNTRACKERS-NEXT: {{ $}}
; GFX908-GCNTRACKERS-NEXT: bb.1:
; GFX908-GCNTRACKERS-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 255
; GFX908-GCNTRACKERS-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 [[S_MOV_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_9]]
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF29]]
- ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_28]], implicit [[V_CVT_I32_F32_e32_30]], implicit [[DEF30]]
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF29]]
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF30]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_10]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_11]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_12]]
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index edf020cce0fcc..a76318434a391 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -7,6 +7,14 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10_3 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX7-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_1-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_3-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX11-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX12-GCNTRACKERS %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
@@ -564,6 +572,541 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX7-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s32, 6
+; GFX7-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX7-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0x4040
+; GFX7-GCNTRACKERS-NEXT: v_mad_u32_u24 v0, v0, 64, s32
+; GFX7-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 6, v0
+; GFX7-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX8-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, 64, v0
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX8-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0x4040
+; GFX8-GCNTRACKERS-NEXT: v_mad_u32_u24 v0, v0, 64, s32
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 6, v0
+; GFX8-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 64, v0
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 0x4040, v0
+; GFX900-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4044
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v23, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX942-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 0x4040
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4044
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v23, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v24, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v24, 0x4040, v24
+; GFX10_1-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v24
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v24, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v24, 0x4040, v24
+; GFX10_3-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v24
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4044
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v23, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 0x4040
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4044
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v23, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v23, s32 offset:16388 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX12-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s32
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: s_add_co_ci_u32 s59, s32, 0x4000
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX12-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v23, off, s32 offset:16388 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
%alloca1 = alloca i32, align 4, addrspace(5)
call void asm sideeffect "; use alloca0 $0", "v"(ptr addrspace(5) %alloca0)
@@ -1085,6 +1628,485 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_mad_u32_u24 v22, 16, 64, s32
+; GFX7-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v22, 6, v22
+; GFX7-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_mad_u32_u24 v22, 16, 64, s32
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v22, 6, v22
+; GFX8-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 6, s32
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v22, 16, v22
+; GFX900-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4010
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v21, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 16
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4010
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v21, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v22, 16, v22
+; GFX10_1-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v22, 16, v22
+; GFX10_3-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4010
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v21, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 16
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4010
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v21, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v21, s32 offset:16384 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s32
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v21, off, s32 offset:16384 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 16, addrspace(5)
; Force no SGPRs to be available for the carry-out of the vector add.
@@ -1661,6 +2683,543 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s28, 17
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s29, 18
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX7-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX7-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s32, 6
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX7-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX7-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s28, v23, 17
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s29, v23, 18
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX8-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX8-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX8-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, 64, v0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX900-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX900-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 64, v0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x8040
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v22, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX942-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s58, s32, 0x4240
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x8040
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v22, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: s_lshr_b32 s4, s32, 5
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s58, s4, 0x4240
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: s_lshr_b32 s4, s32, 5
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s58, s4, 0x4240
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x8040
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v22, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s58, s32, 0x4240
+; GFX11-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x8040
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v22, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v22, s32 offset:32768 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX12-GCNTRACKERS-NEXT: s_add_co_i32 s58, s32, 0x4200
+; GFX12-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s32
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v22, off, s32 offset:32768 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
%alloca1 = alloca [4096 x i32], align 4, addrspace(5)
call void asm sideeffect "; use alloca0 $0", "v"(ptr addrspace(5) %alloca0)
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
new file mode 100644
index 0000000000000..516db40a5af76
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
@@ -0,0 +1,43 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+
+# Test that the upward register pressure tracker accounts for early-clobber
+# physical register defs overlapping with physical register uses.
+# With physreg tracking, the EC def s[10:11] (2 SGPRs) overlaps with the use
+# s12 (1 SGPR), producing higher max pressure at the INLINEASM instruction.
+
+---
+name: ec_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: ec_physreg
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ; RPU: 3 1
+ ; RPU: 5 1 INLINEASM &"s_mov_b64 $0, $1" {{.*}} early-clobber $sgpr10_sgpr11
+ ; RPU: 4 1
+ ;
+ ; RPU-NOPHYS-LABEL: name: ec_physreg
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, $1" {{.*}} early-clobber $sgpr10_sgpr11
+ ; RPU-NOPHYS: 2 1
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ INLINEASM &"s_mov_b32 $0, 42", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr12
+ %3:sreg_32 = COPY $sgpr12
+ $sgpr12 = COPY %3
+ INLINEASM &"s_mov_b64 $0, $1", 1 /* sideeffect attdialect */, 11 /* regdef-ec */, implicit-def early-clobber $sgpr10_sgpr11, 9 /* reguse */, $sgpr12
+ %4:sreg_64 = COPY $sgpr10_sgpr11
+ %5:sreg_32 = COPY %4.sub0
+ %6:vgpr_32 = COPY %5
+ GLOBAL_STORE_DWORD_SADDR %2, %6, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
new file mode 100644
index 0000000000000..bc6f1e1888bbc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
@@ -0,0 +1,239 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s 2>&1 | FileCheck %s --check-prefix=RPU
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward %s 2>&1 | FileCheck %s --check-prefix=RPD
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
+
+# Tests for physical register pressure tracking edge cases:
+# 1. Non-allocatable registers (e.g. $scc) should not affect pressure.
+# 2. Aliasing: a tuple def ($sgpr10_sgpr11) does not kill a live sub-register
+# ($sgpr11) tracked under a different name, leading to over-counted pressure.
+
+# Non-allocatable physical register should not change pressure.
+---
+name: nonallocatable_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: nonallocatable_physreg
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ; RPU: 2 1
+ ; RPU: 2 1 S_NOP 0, implicit-def $scc
+ ; RPU: 2 1
+ ;
+ ; RPU-NOPHYS-LABEL: name: nonallocatable_physreg
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 S_NOP 0, implicit-def $scc
+ ; RPU-NOPHYS: 2 1
+ ;
+ ; RPD-LABEL: name: nonallocatable_physreg
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Initial pressure includes $sgpr8_sgpr9 live-in (2 SGPR units).
+ ;
+ ; RPD: 2 0
+ ; RPD: 2 1
+ ; RPD: 2 1 S_NOP 0, implicit-def $scc
+ ; RPD: 2 1
+ ;
+ ; RPD-NOPHYS-LABEL: name: nonallocatable_physreg
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-in physical register pressure.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 2 1 S_NOP 0, implicit-def $scc
+ ; RPD-NOPHYS: 2 1
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ S_NOP 0, implicit-def $scc
+ GLOBAL_STORE_DWORD_SADDR %2, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
+
+# Aliasing: $sgpr10_sgpr11 is defined, then $sgpr11 (a sub-register) is used.
+# Unit-level tracking correctly handles this: the upward tracker adds
+# $sgpr11's unit when processing the use, and when receding past the def of
+# $sgpr10_sgpr11, it finds and removes $sgpr11's unit, correctly decrementing
+# pressure.
+---
+name: aliased_physreg_tuple_def_subreg_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ;
+ ; The def of $sgpr10_sgpr11 correctly kills $sgpr11's unit via
+ ; unit-level tracking, reducing SGPR pressure from 3 to 2.
+ ;
+ ; RPU: 2 1
+ ; RPU: 3 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPU: 3 1
+ ; RPU: 3 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPU-NOPHYS-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no physical register pressure is counted,
+ ; so the INLINEASM shows no changes in pressure.
+ ;
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 3 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPD-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Initial pressure includes $sgpr8_sgpr9 live-in (2 SGPR units).
+ ; Downward tracker: the INLINEASM def adds $sgpr10_sgpr11 (2 units =
+ ; 2 SGPRs). At the COPY use of $sgpr11, the corresponding reg-unit
+ ; is not found in the cached live ranges, so it conservatively keeps
+ ; the unit live. The virtual register def leads to the increase in
+ ; pressure from 4 to 5.
+ ;
+ ; RPD: 2 0
+ ; RPD: 2 1
+ ; RPD: 4 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPD: 4 1
+ ; RPD: 5 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPD-NOPHYS-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-in physical register pressure
+ ; and no physical register pressure changes at instructions.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 3 1 %3:sreg_32 = COPY $sgpr11
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ INLINEASM &"s_mov_b64 $0, 0", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr10_sgpr11
+ %3:sreg_32 = COPY $sgpr11
+ %4:vgpr_32 = COPY %3
+ GLOBAL_STORE_DWORD_SADDR %2, %4, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
+
+# Live-out seeding: bb.0 defines $sgpr10 via INLINEASM and branches to bb.1.
+# bb.1 has $sgpr10 as a live-in. The upward tracker for bb.0 is initialized
+# with bb.0's live-outs (= successor bb.1's live-ins). Without live-out
+# seeding, the upward tracker treats the INLINEASM def as dead and never
+# accounts for $sgpr10's pressure in the region below the def.
+---
+name: physreg_liveout
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: physreg_liveout
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ;
+ ; Top of bb.0: after receding past all instructions, $sgpr8_sgpr9 remains
+ ; live from the COPY use. SGPR=2 ($sgpr8_sgpr9 = 2 reg-units).
+ ;
+ ; RPU: 2 0
+ ; RPU: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPU: 2 0
+ ; RPU: 2 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPU: 2 0
+ ;
+ ; Below the INLINEASM def, $sgpr10 is live (from live-out seeding),
+ ; so SGPR=3 (%1 = 2 units + $sgpr10 = 1 unit).
+ ;
+ ; RPU: 3 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPU: 3 0
+ ; RPU: 3 0 S_BRANCH %bb.1
+ ; RPU: 3 0
+ ;
+ ; RPU-NOPHYS-LABEL: name: physreg_liveout
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-out seeding: the INLINEASM def is dead
+ ; from the upward tracker's perspective, so $sgpr10 is never counted.
+ ; $sgpr8_sgpr9 is also not tracked, so top pressure is 0.
+ ;
+ ; RPU-NOPHYS: 0 0
+ ; RPU-NOPHYS: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 S_BRANCH %bb.1
+ ; RPU-NOPHYS: 2 0
+ ;
+ ; RPD-LABEL: name: physreg_liveout
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Downward tracker: initial pressure includes $sgpr8_sgpr9 live-in (2).
+ ; INLINEASM adds $sgpr10 (1 unit), bringing pressure to 3.
+ ;
+ ; RPD: 2 0
+ ; RPD: 4 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPD: 2 0
+ ; RPD: 4 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPD: 2 0
+ ; RPD: 3 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPD: 3 0
+ ; RPD: 3 0 S_BRANCH %bb.1
+ ; RPD: 3 0
+ ;
+ ; RPD-NOPHYS-LABEL: name: physreg_liveout
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking: no live-in pressure, INLINEASM has no effect.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 4 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 2 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 2 0 S_BRANCH %bb.1
+ ; RPD-NOPHYS: 2 0
+ bb.0:
+ liveins: $sgpr8_sgpr9
+ successors: %bb.1
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ INLINEASM &"s_mov_b32 $0, 0", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr10
+ S_BRANCH %bb.1
+
+ bb.1:
+ liveins: $sgpr10
+
+ %2:vgpr_32 = COPY $sgpr10
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %3, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
new file mode 100644
index 0000000000000..d6e3b6a734eec
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
@@ -0,0 +1,68 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
+# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
+# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
+
+# Test that physical register live-ins from MBB liveins are correctly
+# included in per-region pressure when using GCN trackers in the scheduler.
+#
+# The function has two scheduling regions separated by SCHED_BARRIER.
+# $sgpr10 is live-in to the MBB but only used in the second region.
+# With physical register tracking, $sgpr10 should contribute 1 SGPR
+# of pressure in the first region (live-through). Without physical
+# tracking, it should not appear.
+
+--- |
+ define amdgpu_kernel void @physreg_livein_across_regions() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx900" }
+...
+
+# GCN-LABEL: physreg_livein_across_regions
+#
+# Region 0 (bottom of block, processed first by scheduler):
+# $sgpr10 is used here via COPY. With phys tracking, $sgpr10 contributes
+# 1 extra SGPR compared to NOPHYS.
+# GCN: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 3
+# GCN: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 3
+#
+# Region 1 (top of block, processed second):
+# $sgpr10 is live-through (MBB live-in, not used until region 0).
+# Physical live-in tracking captures it in per-region pressure.
+# GCN: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 5
+# GCN: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 5
+
+# GCN-NOPHYS-LABEL: physreg_livein_across_regions
+#
+# Region 0: $sgpr10 not tracked, only virtual regs contribute.
+# GCN-NOPHYS: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 2
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 2
+#
+# Region 1: No physical register pressure.
+# GCN-NOPHYS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 4
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 4
+
+---
+name: physreg_livein_across_regions
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5, $sgpr10
+
+ ; Region 1: virtual reg defs only, $sgpr10 is live-through.
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ SCHED_BARRIER 0
+
+ ; Region 2: uses $sgpr10.
+ %2:vgpr_32 = COPY $sgpr10
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %3, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
new file mode 100644
index 0000000000000..0be216846f4be
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
@@ -0,0 +1,84 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
+# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
+# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
+
+# Test that physical register live-outs (derived from successor block liveins)
+# are correctly included in per-region pressure when using GCN trackers.
+#
+# bb.0 defines $sgpr10 in the first scheduling region and branches to bb.1
+# which has $sgpr10 as a live-in. This makes $sgpr10 a live-out of bb.0.
+# The second region in bb.0 does not touch $sgpr10, so it is live-through.
+# With physical register tracking, $sgpr10 should contribute 1 SGPR of
+# pressure in that second region. Without physical tracking, it should not.
+
+--- |
+ define amdgpu_kernel void @physreg_liveout_across_regions() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx900" }
+...
+
+# GCN-LABEL: physreg_liveout_across_regions
+#
+# Region 0 (bottom of bb.0, processed first by scheduler):
+# $sgpr10 is live-through here (defined above, live-out to bb.1).
+# With phys tracking it contributes +1 SGPR compared to NOPHYS.
+# GCN: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5
+# GCN: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5
+#
+# Region 1 (top of bb.0, processed second):
+# $sgpr4_sgpr5 live-in (+2) and $sgpr10 def (+1) add physical pressure.
+# GCN: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 4
+# GCN: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 4
+#
+# bb.1: $sgpr10 is a physical live-in here.
+# GCN: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 1
+# GCN: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 1
+
+# GCN-NOPHYS-LABEL: physreg_liveout_across_regions
+#
+# Region 0: $sgpr10 live-out not tracked, only virtual regs contribute.
+# GCN-NOPHYS: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 4
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 4
+#
+# Region 1: No physical register pressure.
+# GCN-NOPHYS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 2
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 2
+#
+# bb.1: $sgpr10 live-in not tracked.
+# GCN-NOPHYS: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 0
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 0
+
+---
+name: physreg_liveout_across_regions
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ successors: %bb.1
+
+ ; Region 1: defines $sgpr10, which is live-out to bb.1.
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ $sgpr10 = S_MOV_B32 42
+ SCHED_BARRIER 0
+
+ ; Region 0: $sgpr10 is live-through (defined above, live-out to bb.1).
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %2, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_BRANCH %bb.1
+
+ bb.1:
+ liveins: $sgpr10
+
+ %3:vgpr_32 = COPY $sgpr10
+ %4:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %4, %3, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index f70cd6816a966..b2b73e9a96fcb 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,5 +1,6 @@
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=ERR-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
@@ -16,10 +17,13 @@
i64 ; vcc
}
-; ERR-GCNTRACKERS: ran out of registers during register allocation
+; GCN-TRACKERS-NOT: ran out of registers during register allocation
; GCN-NOT: ran out of registers during register allocation
+; GCN-NOPHYS-FAIL: ran out of registers during register allocation
-; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
+; GCN Trackers now track physical register pressure correctly, so this test
+; verifies that both trackers can successfully handle code with heavy physical
+; register usage from inline assembly.
define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0 {
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index d1c972758692f..91a071dd69049 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -1,17 +1,32 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 < %s | FileCheck --check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck --check-prefix=GCN-GCNTRACKERS %s
-
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
+; RUN: FileCheck --check-prefix=SCHED %s < %t
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-GCNTRACKERS %s
+; RUN: FileCheck --check-prefix=SCHED-GCNTRACKERS %s < %t
+; REQUIRES: asserts
; CHECK-LABEL: {{^}}spill:
; GCN: NumSgprs: 104
; GCN-GCNTRACKERS: NumSgprs: 104
; GCN: NumVgprs: 1
-; GCN-GCNTRACKERS: NumVgprs: 2
+; GCN-GCNTRACKERS: NumVgprs: 1
; GCN: ScratchSize: 0
; GCN-GCNTRACKERS: ScratchSize: 0
; GCN: Occupancy: 4
; GCN-GCNTRACKERS: Occupancy: 4
-
-; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
+;
+; Check scheduling pressure values:
+; SCHED-LABEL: spill:%bb.0 entry
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 98
+; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 97
+;
+; SCHED-GCNTRACKERS-LABEL: spill:%bb.0 entry
+; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
+; SCHED-GCNTRACKERS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
+;
+; NOTE: GCN Trackers now track pressure from both virtual and physical registers.
+; The GCN tracker now matches the generic tracker's VGPR count (1 VGPR).
+; When a live range is not found for a physical regunit, we conservatively
+; assume the unit is live, so Region SGPR pressure can be higher (193 vs 98).
+; Pressure after scheduling remains 98 vs 97 due to physical register tracking.
define amdgpu_kernel void @spill(ptr addrspace(1) %arg, i32 %cnd) #0 {
entry:
@@ -247,9 +262,15 @@ bb3:
; GCN: NumSgprs: 104
; GCN-GCNTRACKERS: NumSgprs: 104
; GCN: NumVgprs: 2
-; GCN-GCNTRACKERS: NumVgprs: 3
+; GCN-GCNTRACKERS: NumVgprs: 2
; GCN: ScratchSize: 8
-; GCN-GCNTRACKERS: ScratchSize: 12
+; GCN-GCNTRACKERS: ScratchSize: 8
+;
+; SCHED-LABEL: spill_func:%bb.0 entry
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 97
+;
+; SCHED-GCNTRACKERS-LABEL: spill_func:%bb.0 entry
+; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
define void @spill_func(ptr addrspace(1) %arg) #0 {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
new file mode 100644
index 0000000000000..7c947f4a942a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
@@ -0,0 +1,630 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
+; RUN: FileCheck --check-prefix=GCN-DEBUG %s < %t
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=NO-GCN %s
+; RUN: FileCheck --check-prefix=GENERIC-DEBUG %s < %t
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
+; RUN: FileCheck --check-prefix=GCN-NOPHYS-DEBUG %s < %t
+; REQUIRES: asserts
+
+; Test that GCN trackers correctly track physical register pressure from inline asm
+
+; GCN-DEBUG-LABEL: test_single_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_single_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_single_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_single_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_single_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_single_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_single_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 $0, 0", "={s10}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test multiple physical registers
+
+; GCN-DEBUG-LABEL: test_multiple_physregs
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_multiple_physregs
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_multiple_physregs
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_multiple_physregs(ptr addrspace(1) %out) {
+; GCN-LABEL: test_multiple_physregs:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_add_i32 s4, s10, s11
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_multiple_physregs:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_add_i32 s4, s10, s11
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s4
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_multiple_physregs:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_add_i32 s4, s10, s11
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %result = call { i32, i32 } asm sideeffect "s_mov_b32 $0, 0; s_mov_b32 $1, 1", "={s10},={s11}"()
+ %r0 = extractvalue { i32, i32 } %result, 0
+ %r1 = extractvalue { i32, i32 } %result, 1
+ %sum = add i32 %r0, %r1
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register with virtual registers
+
+; GCN-DEBUG-LABEL: test_physreg_with_vreg
+; GCN-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 10, LVGPR WT: 0, LSGPR WT: 12
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
+
+; GENERIC-DEBUG-LABEL: test_physreg_with_vreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+
+; GCN-NOPHYS-DEBUG-LABEL: test_physreg_with_vreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+
+define amdgpu_kernel void @test_physreg_with_vreg(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GCN-LABEL: test_physreg_with_vreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s4, s2
+; GCN-NEXT: s_mov_b32 s5, s3
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_physreg_with_vreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s7, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s6, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: s_mov_b32 s4, s0
+; NO-GCN-NEXT: s_mov_b32 s5, s1
+; NO-GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: s_mov_b32 s4, s2
+; NO-GCN-NEXT: s_mov_b32 s5, s3
+; NO-GCN-NEXT: s_waitcnt vmcnt(0)
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_physreg_with_vreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s6, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s0
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s1
+; GCN-NOPHYS-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s2
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s3
+; GCN-NOPHYS-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %asm_val = call i32 asm sideeffect "s_mov_b32 $0, 0", "={s10}"()
+ %val = load i32, ptr addrspace(1) %in
+ %sum = add i32 %asm_val, %val
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; Test early-clobber constraint
+
+; GCN-DEBUG-LABEL: test_early_clobber
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_early_clobber
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_early_clobber(ptr addrspace(1) %out) {
+; GCN-LABEL: test_early_clobber:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_early_clobber:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_early_clobber:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 $0, 0", "=&{s10}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test early-clobber constraint with a tuple (64-bit) register.
+; The input s12 and early-clobber output s[10:11] have distinct live ranges.
+
+; GCN-DEBUG-LABEL: test_early_clobber_tuple
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 9
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GENERIC-DEBUG-LABEL: test_early_clobber_tuple
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber_tuple
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_early_clobber_tuple(ptr addrspace(1) %out) {
+; GCN-LABEL: test_early_clobber_tuple:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s12, 42
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[10:11], s12
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_early_clobber_tuple:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s12, 42
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[10:11], s12
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_early_clobber_tuple:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s12, 42
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[10:11], s12
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %in = call i32 asm sideeffect "s_mov_b32 $0, 42", "={s12}"()
+ %val = call i64 asm sideeffect "s_mov_b64 $0, $1", "=&{s[10:11]},{s12}"(i32 %in)
+ %lo = trunc i64 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register input
+
+; GCN-DEBUG-LABEL: test_physreg_input
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_physreg_input
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_physreg_input
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_physreg_input(ptr addrspace(1) %out) {
+; GCN-LABEL: test_physreg_input:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_physreg_input:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s11
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_physreg_input:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 s10, 5; s_add_u32 $0, s10, 1", "={s11}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register pressure for tuple (64-bit) registers.
+; GCN tracker counts the 2 SGPRs.
+
+; GCN-DEBUG-LABEL: test_tuple_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GENERIC-DEBUG-LABEL: test_tuple_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_tuple_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_tuple_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_tuple_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[10:11], 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_tuple_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[10:11], 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_tuple_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[10:11], 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i64 asm sideeffect "s_mov_b64 $0, 0", "={s[10:11]}"()
+ %lo = trunc i64 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register pressure for 128-bit tuple.
+; GCN tracker counts the 4 SGPRs.
+
+; GCN-DEBUG-LABEL: test_tuple128_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+
+; GENERIC-DEBUG-LABEL: test_tuple128_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GCN-NOPHYS-DEBUG-LABEL: test_tuple128_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+define amdgpu_kernel void @test_tuple128_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_tuple128_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_tuple128_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s8
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_tuple128_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i128 asm sideeffect "s_mov_b64 $0, 0; s_mov_b64 $0+2, 0", "={s[8:11]}"()
+ %lo = trunc i128 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test virtual and physical register overlap
+
+; GCN-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GCN-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 16, LVGPR WT: 0, LSGPR WT: 16
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
+
+; GENERIC-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+
+; GCN-NOPHYS-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+
+define amdgpu_kernel void @test_vreg_and_physreg_live_range_overlap(ptr addrspace(1) %in1, ptr addrspace(1) %in2, ptr addrspace(1) %out) {
+; GCN-LABEL: test_vreg_and_physreg_live_range_overlap:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_mov_b32 s0, s2
+; GCN-NEXT: s_mov_b32 s1, s3
+; GCN-NEXT: s_mov_b32 s2, s6
+; GCN-NEXT: s_mov_b32 s3, s7
+; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s14, s6
+; GCN-NEXT: s_mov_b32 s15, s7
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_vreg_and_physreg_live_range_overlap:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; NO-GCN-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; NO-GCN-NEXT: s_mov_b32 s7, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s6, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: s_mov_b32 s4, s0
+; NO-GCN-NEXT: s_mov_b32 s5, s1
+; NO-GCN-NEXT: s_mov_b32 s0, s2
+; NO-GCN-NEXT: s_mov_b32 s1, s3
+; NO-GCN-NEXT: s_mov_b32 s2, s6
+; NO-GCN-NEXT: s_mov_b32 s3, s7
+; NO-GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; NO-GCN-NEXT: s_mov_b32 s14, s6
+; NO-GCN-NEXT: s_mov_b32 s15, s7
+; NO-GCN-NEXT: s_waitcnt vmcnt(0)
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_vreg_and_physreg_live_range_overlap:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; GCN-NOPHYS-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s6, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s0
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s1
+; GCN-NOPHYS-NEXT: s_mov_b32 s0, s2
+; GCN-NOPHYS-NEXT: s_mov_b32 s1, s3
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, s6
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, s7
+; GCN-NOPHYS-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_mov_b32 s14, s6
+; GCN-NOPHYS-NEXT: s_mov_b32 s15, s7
+; GCN-NOPHYS-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %result = call { i32, i32 } asm sideeffect "s_mov_b32 $0, 0; s_mov_b32 $1, 1", "={s10},={s11}"()
+ %val1 = load i32, ptr addrspace(1) %in1
+ %val2 = load i32, ptr addrspace(1) %in2
+ %sum = add i32 %val1, %val2
+ %r0 = extractvalue { i32, i32 } %result, 0
+ %r1 = extractvalue { i32, i32 } %result, 1
+ %with_asm = add i32 %sum, %r0
+ %final = add i32 %with_asm, %r1
+ store i32 %final, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
index f31fed5b6a88a..6150e04c045c3 100644
--- a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
+++ b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
@@ -59,7 +59,8 @@ body: |
// Track pressure across MBB1.
{
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// There is a non-debug instruction in bb.1 (%2's def), so advance should
// return true.
@@ -75,7 +76,8 @@ body: |
// Track pressure just across the first debug value of bb.1.
{
MachineBasicBlock::iterator Dbg1 = std::next(MBB1.begin());
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// The following unpacks a call to
// advance(*MBB1.begin(), Dbg1, [MBB1LiveIns|nullptr])
@@ -126,7 +128,8 @@ body: |
LIS, MF.getRegInfo());
MachineBasicBlock &MBB1 = *MF.getBlockNumbered(1);
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// The following unpacks a call to
// advance(MBB1.begin(), MBB1.end(), [MBB1LiveIns|nullptr])
>From 33fc5afdb35053cc7f6056601bc49518ef0aea16 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Fri, 3 Jul 2026 20:29:37 -0500
Subject: [PATCH 2/2] De-coupled physical register tracking from GCN trackers.
Physical register tracking is enabled when GCNRPTracker is used.
It can be disabled globally with -amdgpu-track-physregs-in-gcn-trackers=false.
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 31 +-
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 21 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 14 -
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 414 ++++++-------
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 350 +++++------
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 478 +++++++--------
llvm/test/CodeGen/AMDGPU/addrspacecast.ll | 12 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 346 ++++++++---
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 462 +++++++-------
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 188 +++---
.../CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll | 398 ++++++------
.../CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll | 574 +++++++++---------
...ffer-fat-pointers-contents-legalization.ll | 74 +--
.../CodeGen/AMDGPU/call-argument-types.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 10 +-
...rval-bug-in-rename-independent-subregs.mir | 8 +-
llvm/test/CodeGen/AMDGPU/memory_clause.mir | 3 +-
.../AMDGPU/preserve-wwm-copy-dst-reg.ll | 8 +-
.../regpressure-physreg-early-clobber.mir | 2 +-
.../AMDGPU/regpressure-physreg-limits.mir | 4 +-
.../CodeGen/AMDGPU/regpressure_printer.mir | 156 ++---
.../CodeGen/AMDGPU/sched-physreg-liveins.mir | 2 +-
.../CodeGen/AMDGPU/sched-physreg-liveouts.mir | 2 +-
.../schedule-amdgpu-tracker-physreg-crash.ll | 2 +-
.../AMDGPU/schedule-amdgpu-tracker-physreg.ll | 17 +-
.../AMDGPU/schedule-gcn-physreg-pressure.ll | 36 +-
.../CodeGen/AMDGPU/soft-clause-dbg-value.mir | 1 -
.../AMDGPU/splitkit-getsubrangeformask.ll | 11 -
28 files changed, 1889 insertions(+), 1741 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index c9053c3ce93be..55df93886ea3b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -22,6 +22,17 @@ using namespace llvm;
#define DEBUG_TYPE "machine-scheduler"
+static cl::opt<bool> TrackPhysRegInGCNTrackers(
+ "amdgpu-track-physregs-in-gcn-trackers", cl::Hidden,
+ cl::desc("Track physical registers (e.g. from inline asm) in the GCN "
+ "Up/Down RP trackers. Enabled by default; acts as a global "
+ "safety switch to control physical register pressure tracking."),
+ cl::init(true));
+
+bool GCNRPTracker::physRegTrackingEnabled() const {
+ return TrackPhysRegInGCNTrackers;
+}
+
bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
const GCNRPTracker::LiveRegSet &S2) {
if (S1.size() != S2.size())
@@ -642,9 +653,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
VirtLiveRegs = llvm::getVirtLiveRegs(SI, LIS, MRI);
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
- updatePhysRegTracking();
- // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
- // stale data if physical tracking was previously enabled.
+ // Always clear PhysLiveRegUnits even when physical tracking is disabled, to
+ // avoid stale data if physical tracking was previously enabled.
PhysLiveRegUnits.reset();
}
@@ -657,9 +667,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
this->VirtLiveRegs = VirtLiveRegs;
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
- updatePhysRegTracking();
- // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
- // stale data if physical tracking was previously enabled.
+ // Always clear PhysLiveRegUnits even when physical tracking is disabled, to
+ // avoid stale data if physical tracking was previously enabled.
PhysLiveRegUnits.reset();
}
@@ -671,7 +680,7 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRInfo,
}
void GCNRPTracker::initPhysLiveUnits(const BitVector &PhysLiveUnits) {
- if (!TrackPhysRegs)
+ if (!physRegTrackingEnabled())
return;
PhysLiveRegUnits = PhysLiveUnits;
GCNRegPressure PhysPressure = constructPhysRegPressure();
@@ -753,7 +762,7 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
CurPressure.inc(U.VRegOrUnit.asVirtualReg(), PrevMask, LiveMask, *MRI);
}
- if (TrackPhysRegs) {
+ if (physRegTrackingEnabled()) {
for (const MachineOperand &MO : MI.all_uses()) {
if (!MO.readsReg())
continue;
@@ -804,7 +813,7 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
else
GCNRPTracker::reset(*MI.getParent(), /*End=*/true);
- if (SeedPhysMBB && TrackPhysRegs &&
+ if (SeedPhysMBB && physRegTrackingEnabled() &&
MI.getMF()->getProperties().hasTracksLiveness())
initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveins());
@@ -1025,7 +1034,7 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
LaneBitmask NewMask = LiveMask & ~LastUseMask;
TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
- } else if (TrackPhysRegs) {
+ } else if (physRegTrackingEnabled()) {
MCRegUnit Unit = Use.VRegOrUnit.asMCRegUnit();
unsigned U = static_cast<unsigned>(Unit);
if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SlotIdx))
@@ -1043,7 +1052,7 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
LaneBitmask NewMask = LiveMask | Def.LaneMask;
TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
- } else if (TrackPhysRegs) {
+ } else if (physRegTrackingEnabled()) {
MCRegUnit Unit = Def.VRegOrUnit.asMCRegUnit();
unsigned U = static_cast<unsigned>(Unit);
if (!PhysLiveRegUnits.test(U))
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 92819b15eaaf2..de2c0ad2f825b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -354,18 +354,12 @@ class GCNRPTracker {
GCNRegPressure CurPressure, MaxPressure;
- // Flag to control whether physical register tracking is active.
- // Set to true when GCNTrackers are enabled, false otherwise.
- bool TrackPhysRegs = false;
-
const MachineInstr *LastTrackedMI = nullptr;
GCNRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
: LIS(LIS), MRI(&MRI),
SRI(static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo())),
- PhysLiveRegUnits(SRI->getNumRegUnits()) {
- updatePhysRegTracking();
- }
+ PhysLiveRegUnits(SRI->getNumRegUnits()) {}
/// Resets tracker before or \p After the provided \p MI, which can be a debug
/// instruction.
@@ -383,7 +377,8 @@ class GCNRPTracker {
LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;
bool shouldTrackPhysReg(Register Reg) const {
- return TrackPhysRegs && Reg.isPhysical() && MRI->isAllocatable(Reg);
+ return physRegTrackingEnabled() && Reg.isPhysical() &&
+ MRI->isAllocatable(Reg);
}
// Check if a register unit is live at a given slot index per LIS.
@@ -405,9 +400,8 @@ class GCNRPTracker {
GCNRegPressure &Pressure);
public:
- // Enable physical register tracking only if both GCNTrackers and
- // TrackPhysRegInTrackers are true.
- void updatePhysRegTracking();
+ // Returns whether physical register tracking is enabled.
+ bool physRegTrackingEnabled() const;
/// Resets tracker with the provided \p VirtLiveRegs.
void reset(const MachineRegisterInfo &MRI, const LiveRegSet &VirtLiveRegs);
@@ -425,7 +419,8 @@ class GCNRPTracker {
/// and update CurPressure/MaxPressure accordingly.
template <typename RangeT>
void initPhysLiveUnitsFromRegMaskPairs(RangeT &&Pairs) {
- assert(TrackPhysRegs && "physical register tracking must be enabled");
+ assert(physRegTrackingEnabled() &&
+ "physical register tracking must be enabled");
for (const auto &RM : Pairs)
if (MRI->isAllocatable(RM.PhysReg))
addUnitsAndIncPressure(RM.PhysReg, CurPressure);
@@ -463,7 +458,7 @@ class GCNUpwardRPTracker : public GCNRPTracker {
void reset(const MachineRegisterInfo &MRI, SlotIndex SI,
const MachineBasicBlock *SeedPhysMBB = nullptr) {
GCNRPTracker::reset(MRI, llvm::getVirtLiveRegs(SI, LIS, MRI));
- if (SeedPhysMBB && TrackPhysRegs &&
+ if (SeedPhysMBB && physRegTrackingEnabled() &&
SeedPhysMBB->getParent()->getProperties().hasTracksLiveness())
initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveouts());
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 5e30603dc9383..078a745ee9bae 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -78,12 +78,6 @@ static cl::opt<bool> GCNTrackers(
cl::desc("Use the AMDGPU specific RPTrackers during scheduling"),
cl::init(false));
-static cl::opt<bool> TrackPhysRegInTrackers(
- "amdgpu-trackers-physical-register-tracking", cl::Hidden,
- cl::desc("When using GCN trackers, count physical registers (e.g. from "
- "inline asm) in pressure."),
- cl::init(true));
-
static cl::opt<unsigned> PendingQueueLimit(
"amdgpu-scheduler-pending-queue-limit", cl::Hidden,
cl::desc(
@@ -214,14 +208,6 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
<< ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
}
-void GCNRPTracker::updatePhysRegTracking() {
- if (!GCNTrackers || !TrackPhysRegInTrackers) {
- TrackPhysRegs = false;
- return;
- }
- TrackPhysRegs = true;
-}
-
/// Checks whether \p SU can use the cached DAG pressure diffs to compute the
/// current register pressure.
///
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 9d3302a996ebd..7b62737a79bb7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -8137,250 +8137,250 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6-LABEL: v_fshl_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshr_b64 v[23:24], v[8:9], 1
; GFX6-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX6-NEXT: v_not_b32_e32 v18, 63
-; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
-; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v18
-; GFX6-NEXT: v_lshr_b64 v[23:24], v[0:1], v23
-; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], v19
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v27
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 31, v10
+; GFX6-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
+; GFX6-NEXT: v_or_b32_e32 v24, v24, v8
+; GFX6-NEXT: v_mov_b32_e32 v8, 0x7f
+; GFX6-NEXT: v_lshr_b64 v[17:18], v[0:1], v17
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v19
+; GFX6-NEXT: v_lshr_b64 v[9:10], v[10:11], 1
+; GFX6-NEXT: v_bfi_b32 v11, v16, 0, v8
+; GFX6-NEXT: v_sub_i32_e32 v16, vcc, 64, v11
+; GFX6-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[9:10], v16
+; GFX6-NEXT: v_lshr_b64 v[25:26], v[23:24], v11
+; GFX6-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX6-NEXT: v_or_b32_e32 v22, v25, v16
+; GFX6-NEXT: v_or_b32_e32 v25, v26, v17
+; GFX6-NEXT: v_not_b32_e32 v26, 63
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v19, v26
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[0:1], v16
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[8:9], 1
-; GFX6-NEXT: v_mov_b32_e32 v17, 0x7f
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 31, v10
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], 1
-; GFX6-NEXT: v_bfi_b32 v10, v16, 0, v17
-; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
-; GFX6-NEXT: v_add_i32_e32 v16, vcc, v10, v18
-; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v10
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], v10
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[0:1], v10
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v21
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], v16
-; GFX6-NEXT: v_or_b32_e32 v10, v10, v21
-; GFX6-NEXT: v_or_b32_e32 v11, v11, v22
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v20
-; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[4:5], v10
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[6:7], v16
-; GFX6-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX6-NEXT: v_add_i32_e32 v19, vcc, v16, v18
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v16
-; GFX6-NEXT: v_or_b32_e32 v16, v10, v21
-; GFX6-NEXT: v_or_b32_e32 v21, v11, v22
-; GFX6-NEXT: v_lshl_b64 v[10:11], v[4:5], v19
-; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[12:13], 1
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 31, v14
-; GFX6-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[14:15], 1
-; GFX6-NEXT: v_bfi_b32 v14, v20, 0, v17
-; GFX6-NEXT: v_add_i32_e32 v18, vcc, v14, v18
-; GFX6-NEXT: v_sub_i32_e32 v16, vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v18, v17, v3, s[4:5]
+; GFX6-NEXT: v_add_i32_e64 v2, s[4:5], v11, v26
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[9:10], v2
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v19
+; GFX6-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v11
+; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v25, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[9:10], v11
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX6-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX6-NEXT: v_and_b32_e32 v19, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, v0, s[4:5]
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 64, v19
+; GFX6-NEXT: v_or_b32_e32 v0, v17, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v16, v9
+; GFX6-NEXT: v_lshr_b64 v[9:10], v[4:5], v3
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[6:7], v19
+; GFX6-NEXT: v_or_b32_e32 v3, v18, v11
+; GFX6-NEXT: v_or_b32_e32 v11, v9, v16
+; GFX6-NEXT: v_or_b32_e32 v16, v10, v17
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v19, v26
+; GFX6-NEXT: v_lshl_b64 v[9:10], v[4:5], v19
+; GFX6-NEXT: v_lshl_b64 v[4:5], v[4:5], v17
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v18, 0, v10, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v19, v5, v7, vcc
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[12:13], 1
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 31, v14
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], 1
+; GFX6-NEXT: v_bfi_b32 v14, v20, 0, v8
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v14
+; GFX6-NEXT: v_add_i32_e32 v15, vcc, v14, v26
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[4:5], v14
+; GFX6-NEXT: v_lshl_b64 v[10:11], v[6:7], v10
+; GFX6-NEXT: v_lshr_b64 v[12:13], v[6:7], v14
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[6:7], v15
+; GFX6-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX6-NEXT: v_or_b32_e32 v9, v9, v11
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GFX6-NEXT: v_lshr_b64 v[12:13], v[10:11], v14
-; GFX6-NEXT: v_lshr_b64 v[14:15], v[8:9], v14
-; GFX6-NEXT: v_lshl_b64 v[16:17], v[10:11], v16
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[10:11], v18
-; GFX6-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX6-NEXT: v_or_b32_e32 v15, v15, v17
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v24, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v25, v1
-; GFX6-NEXT: v_or_b32_e32 v3, v23, v3
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v8
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v9
-; GFX6-NEXT: v_or_b32_e32 v6, v6, v10
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v11
+; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v12, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc
+; GFX6-NEXT: v_or_b32_e32 v4, v17, v4
+; GFX6-NEXT: v_or_b32_e32 v5, v18, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v16, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v19, v7
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshrrev_b64 v[23:24], 1, v[8:9]
; GFX8-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX8-NEXT: v_not_b32_e32 v18, 63
-; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
-; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v18
-; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v19, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v27, v[0:1]
+; GFX8-NEXT: v_lshlrev_b32_e32 v8, 31, v10
+; GFX8-NEXT: v_sub_u32_e32 v17, vcc, 64, v19
+; GFX8-NEXT: v_or_b32_e32 v24, v24, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, 0x7f
+; GFX8-NEXT: v_lshrrev_b64 v[17:18], v17, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v19, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[9:10], 1, v[10:11]
+; GFX8-NEXT: v_bfi_b32 v11, v16, 0, v8
+; GFX8-NEXT: v_sub_u32_e32 v16, vcc, 64, v11
+; GFX8-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[9:10]
+; GFX8-NEXT: v_lshrrev_b64 v[25:26], v11, v[23:24]
+; GFX8-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX8-NEXT: v_or_b32_e32 v22, v25, v16
+; GFX8-NEXT: v_or_b32_e32 v25, v26, v17
+; GFX8-NEXT: v_not_b32_e32 v26, 63
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v19, v26
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v17, 0x7f
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 31, v10
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[10:11]
-; GFX8-NEXT: v_bfi_b32 v10, v16, 0, v17
-; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
-; GFX8-NEXT: v_add_u32_e32 v16, vcc, v10, v18
-; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v10
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], v10, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v21
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v22
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v20
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[4:5]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v16, v[6:7]
-; GFX8-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX8-NEXT: v_add_u32_e32 v19, vcc, v16, v18
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v16, v[4:5]
-; GFX8-NEXT: v_or_b32_e32 v16, v10, v21
-; GFX8-NEXT: v_or_b32_e32 v21, v11, v22
-; GFX8-NEXT: v_lshlrev_b64 v[10:11], v19, v[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
-; GFX8-NEXT: v_lshlrev_b32_e32 v10, 31, v14
-; GFX8-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], 1, v[14:15]
-; GFX8-NEXT: v_bfi_b32 v14, v20, 0, v17
-; GFX8-NEXT: v_add_u32_e32 v18, vcc, v14, v18
-; GFX8-NEXT: v_sub_u32_e32 v16, vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v18, v17, v3, s[4:5]
+; GFX8-NEXT: v_add_u32_e64 v2, s[4:5], v11, v26
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v2, v[9:10]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX8-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v25, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v11, v[9:10]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX8-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX8-NEXT: v_and_b32_e32 v19, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, v0, s[4:5]
+; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 64, v19
+; GFX8-NEXT: v_or_b32_e32 v0, v17, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v16, v9
+; GFX8-NEXT: v_lshrrev_b64 v[9:10], v3, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v19, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v3, v18, v11
+; GFX8-NEXT: v_or_b32_e32 v11, v9, v16
+; GFX8-NEXT: v_or_b32_e32 v16, v10, v17
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v19, v26
+; GFX8-NEXT: v_lshlrev_b64 v[9:10], v19, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v17, v[4:5]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v18, 0, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v5, v7, vcc
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], 1, v[12:13]
+; GFX8-NEXT: v_lshlrev_b32_e32 v6, 31, v14
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
+; GFX8-NEXT: v_bfi_b32 v14, v20, 0, v8
+; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v14
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v14, v26
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[12:13], v14, v[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v15, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v11
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GFX8-NEXT: v_lshrrev_b64 v[12:13], v14, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[14:15], v14, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v18, v[10:11]
-; GFX8-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX8-NEXT: v_or_b32_e32 v15, v15, v17
-; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v24, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v25, v1
-; GFX8-NEXT: v_or_b32_e32 v3, v23, v3
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v8
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v9
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v10
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, 0, v12, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc
+; GFX8-NEXT: v_or_b32_e32 v4, v17, v4
+; GFX8-NEXT: v_or_b32_e32 v5, v18, v5
+; GFX8-NEXT: v_or_b32_e32 v6, v16, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v19, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v23, 0x7f, v16
+; GFX9-NEXT: v_and_b32_e32 v19, 0x7f, v16
; GFX9-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX9-NEXT: v_sub_u32_e32 v17, 64, v23
-; GFX9-NEXT: v_mov_b32_e32 v24, 0x7f
+; GFX9-NEXT: v_sub_u32_e32 v17, 64, v19
+; GFX9-NEXT: v_mov_b32_e32 v25, 0x7f
; GFX9-NEXT: v_lshrrev_b64 v[17:18], v17, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v19, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v9, v10, 31, v9
; GFX9-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; GFX9-NEXT: v_bfi_b32 v25, v16, 0, v24
-; GFX9-NEXT: v_sub_u32_e32 v16, 64, v25
+; GFX9-NEXT: v_bfi_b32 v26, v16, 0, v25
+; GFX9-NEXT: v_sub_u32_e32 v16, 64, v26
; GFX9-NEXT: v_or_b32_e32 v21, v17, v21
-; GFX9-NEXT: v_or_b32_e32 v22, v18, v22
; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
-; GFX9-NEXT: v_lshrrev_b64 v[18:19], v25, v[8:9]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GFX9-NEXT: v_or_b32_e32 v18, v18, v16
-; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v23
-; GFX9-NEXT: v_or_b32_e32 v19, v19, v17
+; GFX9-NEXT: v_lshrrev_b64 v[23:24], v26, v[8:9]
+; GFX9-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX9-NEXT: v_or_b32_e32 v22, v23, v16
+; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v19
+; GFX9-NEXT: v_or_b32_e32 v23, v24, v17
; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v23, v[0:1]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GFX9-NEXT: v_cndmask_b32_e32 v26, 0, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v21, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v22, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v17, v0, v2, s[4:5]
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v3, s[4:5]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v0, v[10:11]
-; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v1, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v25, v[10:11]
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffc0, v26
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v3, s[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v2, v[10:11]
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v26, v[10:11]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v23, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v0, v26, v2
-; GFX9-NEXT: v_or_b32_e32 v2, v17, v8
-; GFX9-NEXT: v_and_b32_e32 v17, 0x7f, v20
-; GFX9-NEXT: v_cndmask_b32_e64 v19, 0, v1, s[4:5]
-; GFX9-NEXT: v_or_b32_e32 v1, v18, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, 64, v17
+; GFX9-NEXT: v_or_b32_e32 v0, v18, v2
+; GFX9-NEXT: v_or_b32_e32 v2, v16, v8
+; GFX9-NEXT: v_and_b32_e32 v16, 0x7f, v20
+; GFX9-NEXT: v_cndmask_b32_e64 v21, 0, v1, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, 64, v16
; GFX9-NEXT: v_lshrrev_b64 v[8:9], v3, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[10:11], v17, v[6:7]
-; GFX9-NEXT: v_or_b32_e32 v3, v16, v19
-; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v17
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX9-NEXT: v_or_b32_e32 v3, v17, v21
+; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
; GFX9-NEXT: v_or_b32_e32 v10, v8, v10
; GFX9-NEXT: v_or_b32_e32 v11, v9, v11
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], v17, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], v16, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v17, v[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v11, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v4, v6, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
; GFX9-NEXT: v_lshrrev_b64 v[4:5], 1, v[12:13]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v8, v7, vcc
; GFX9-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
-; GFX9-NEXT: v_bfi_b32 v13, v20, 0, v24
+; GFX9-NEXT: v_bfi_b32 v13, v20, 0, v25
; GFX9-NEXT: v_lshl_or_b32 v5, v14, 31, v5
; GFX9-NEXT: v_sub_u32_e32 v10, 64, v13
; GFX9-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
@@ -8398,9 +8398,9 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v4, v16, v4
+; GFX9-NEXT: v_or_b32_e32 v4, v17, v4
; GFX9-NEXT: v_or_b32_e32 v5, v18, v5
-; GFX9-NEXT: v_or_b32_e32 v6, v17, v6
+; GFX9-NEXT: v_or_b32_e32 v6, v16, v6
; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 9d3a2accf3119..dcc3684422f27 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -7848,182 +7848,182 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GFX6-NEXT: v_mov_b32_e32 v18, 0x7f
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], 1
+; GFX6-NEXT: v_mov_b32_e32 v19, 0x7f
+; GFX6-NEXT: v_lshl_b64 v[17:18], v[0:1], 1
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX6-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX6-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX6-NEXT: v_not_b32_e32 v17, 63
-; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
-; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v17
-; GFX6-NEXT: v_lshr_b64 v[23:24], v[21:22], v23
-; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[21:22], v19
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[21:22], v27
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7f, v16
-; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
-; GFX6-NEXT: v_add_i32_e32 v16, vcc, v2, v17
-; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v2
-; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[10:11], v2
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[8:9], v2
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[10:11], v21
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, 64, v27
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[17:18], v0
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v27
+; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX6-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, 64, v16
+; GFX6-NEXT: v_lshl_b64 v[23:24], v[10:11], v0
+; GFX6-NEXT: v_lshr_b64 v[25:26], v[8:9], v16
+; GFX6-NEXT: v_or_b32_e32 v22, v1, v22
+; GFX6-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX6-NEXT: v_not_b32_e32 v25, 63
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v27, v25
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[17:18], v0
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX6-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
+; GFX6-NEXT: v_add_i32_e64 v0, s[4:5], v16, v25
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[10:11], v0
+; GFX6-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX6-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v23, s[4:5]
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v16
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v1, v24, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[6:7]
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[17:18], v27
+; GFX6-NEXT: v_cndmask_b32_e64 v9, v2, v9, s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX6-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
-; GFX6-NEXT: v_or_b32_e32 v21, v2, v21
-; GFX6-NEXT: v_or_b32_e32 v22, v3, v22
; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], v16
-; GFX6-NEXT: v_bfi_b32 v16, v20, 0, v18
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v24, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v25, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v19, v8
-; GFX6-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v9
; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], 1
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX6-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX6-NEXT: v_or_b32_e32 v6, v6, v4
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
-; GFX6-NEXT: v_add_i32_e32 v21, vcc, v16, v17
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[8:9], v10
-; GFX6-NEXT: v_lshl_b64 v[18:19], v[6:7], v16
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 64, v16
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[8:9], v4
+; GFX6-NEXT: v_lshl_b64 v[10:11], v[6:7], v16
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v16, v25
+; GFX6-NEXT: v_or_b32_e32 v10, v4, v10
+; GFX6-NEXT: v_or_b32_e32 v11, v5, v11
; GFX6-NEXT: v_lshl_b64 v[4:5], v[8:9], v16
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[8:9], v21
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[8:9], v17
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX6-NEXT: v_or_b32_e32 v10, v10, v18
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX6-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
-; GFX6-NEXT: v_or_b32_e32 v11, v11, v19
-; GFX6-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
-; GFX6-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v4, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
-; GFX6-NEXT: v_add_i32_e32 v17, vcc, v6, v17
-; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v6
-; GFX6-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[14:15], v6
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[12:13], v6
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[14:15], v8
-; GFX6-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX6-NEXT: v_or_b32_e32 v9, v7, v9
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v17
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX6-NEXT: v_and_b32_e32 v10, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v8, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v9, v5, v7, vcc
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, 64, v10
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[12:13], v10
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[14:15], v6
+; GFX6-NEXT: v_add_i32_e32 v11, vcc, v10, v25
+; GFX6-NEXT: v_or_b32_e32 v16, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v19, v5, v7
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v11
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[14:15], v10
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v16, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
; GFX6-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
; GFX6-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
-; GFX6-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX6-NEXT: v_or_b32_e32 v2, v21, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v22, v3
+; GFX6-NEXT: v_or_b32_e32 v4, v17, v6
; GFX6-NEXT: v_or_b32_e32 v5, v18, v7
-; GFX6-NEXT: v_or_b32_e32 v6, v10, v8
-; GFX6-NEXT: v_or_b32_e32 v7, v11, v9
+; GFX6-NEXT: v_or_b32_e32 v6, v8, v10
+; GFX6-NEXT: v_or_b32_e32 v7, v9, v11
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v18, 0x7f
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], 1, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v19, 0x7f
+; GFX8-NEXT: v_lshlrev_b64 v[17:18], 1, v[0:1]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX8-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX8-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX8-NEXT: v_not_b32_e32 v17, 63
-; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
-; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v17
-; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[21:22]
-; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v19, v[21:22]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v27, v[21:22]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX8-NEXT: v_and_b32_e32 v2, 0x7f, v16
-; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
-; GFX8-NEXT: v_add_u32_e32 v16, vcc, v2, v17
-; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], v2, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v2, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[10:11]
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 64, v27
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, v[17:18]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v27, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX8-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 64, v16
+; GFX8-NEXT: v_lshlrev_b64 v[23:24], v0, v[10:11]
+; GFX8-NEXT: v_lshrrev_b64 v[25:26], v16, v[8:9]
+; GFX8-NEXT: v_or_b32_e32 v22, v1, v22
+; GFX8-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX8-NEXT: v_not_b32_e32 v25, 63
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v27, v25
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v0, v[17:18]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX8-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
+; GFX8-NEXT: v_add_u32_e64 v0, s[4:5], v16, v25
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11]
+; GFX8-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX8-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v23, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v16
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v1, v24, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[6:7]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v27, v[17:18]
+; GFX8-NEXT: v_cndmask_b32_e64 v9, v2, v9, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; GFX8-NEXT: v_or_b32_e32 v21, v2, v21
-; GFX8-NEXT: v_or_b32_e32 v22, v3, v22
; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[10:11]
-; GFX8-NEXT: v_bfi_b32 v16, v20, 0, v18
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v24, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v25, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v19, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
; GFX8-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX8-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX8-NEXT: v_or_b32_e32 v6, v6, v4
-; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
-; GFX8-NEXT: v_add_u32_e32 v21, vcc, v16, v17
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[18:19], v16, v[6:7]
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 64, v16
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v4, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v16, v25
+; GFX8-NEXT: v_or_b32_e32 v10, v4, v10
+; GFX8-NEXT: v_or_b32_e32 v11, v5, v11
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v16, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v21, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v17, v[8:9]
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v18
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX8-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v19
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
-; GFX8-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v4, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
-; GFX8-NEXT: v_add_u32_e32 v17, vcc, v6, v17
-; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], v6, v[14:15]
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v6, v[12:13]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v8, v[14:15]
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX8-NEXT: v_or_b32_e32 v9, v7, v9
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v17, v[14:15]
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX8-NEXT: v_and_b32_e32 v10, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v5, v7, vcc
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 64, v10
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v10, v[12:13]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], v6, v[14:15]
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v10, v25
+; GFX8-NEXT: v_or_b32_e32 v16, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v19, v5, v7
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v11, v[14:15]
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v10, v[14:15]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v16, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
-; GFX8-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX8-NEXT: v_or_b32_e32 v2, v21, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v22, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v17, v6
; GFX8-NEXT: v_or_b32_e32 v5, v18, v7
-; GFX8-NEXT: v_or_b32_e32 v6, v10, v8
-; GFX8-NEXT: v_or_b32_e32 v7, v11, v9
+; GFX8-NEXT: v_or_b32_e32 v6, v8, v10
+; GFX8-NEXT: v_or_b32_e32 v7, v9, v11
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshr_v2i128:
@@ -8033,51 +8033,51 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_mov_b32_e32 v19, 0x7f
; GFX9-NEXT: v_lshlrev_b64 v[17:18], 1, v[0:1]
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX9-NEXT: v_bfi_b32 v23, v16, 0, v19
+; GFX9-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX9-NEXT: v_sub_u32_e32 v0, 64, v23
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v27
; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[17:18]
-; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
-; GFX9-NEXT: v_and_b32_e32 v26, 0x7f, v16
-; GFX9-NEXT: v_or_b32_e32 v24, v0, v21
-; GFX9-NEXT: v_sub_u32_e32 v0, 64, v26
-; GFX9-NEXT: v_or_b32_e32 v25, v1, v22
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[10:11]
-; GFX9-NEXT: v_lshrrev_b64 v[21:22], v26, v[8:9]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GFX9-NEXT: v_or_b32_e32 v21, v21, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v23
-; GFX9-NEXT: v_or_b32_e32 v22, v22, v1
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v27, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX9-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v16
+; GFX9-NEXT: v_lshlrev_b64 v[23:24], v0, v[10:11]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v27
+; GFX9-NEXT: v_or_b32_e32 v22, v1, v22
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[17:18]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v24, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v25, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v0, v2, s[4:5]
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v26
-; GFX9-NEXT: v_lshlrev_b64 v[16:17], v23, v[17:18]
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v1, v3, s[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX9-NEXT: v_lshrrev_b64 v[25:26], v16, v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v16
+; GFX9-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11]
-; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v26
-; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v18, v0, v21, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v21, v1, v22, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v26, v[10:11]
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v18, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v21, v9, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], v27, v[17:18]
+; GFX9-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX9-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX9-NEXT: v_cndmask_b32_e64 v23, v0, v23, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v17, v1, v24, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v16, v[10:11]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v23, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
-; GFX9-NEXT: v_or_b32_e32 v0, v16, v8
-; GFX9-NEXT: v_or_b32_e32 v1, v17, v9
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v8
+; GFX9-NEXT: v_or_b32_e32 v1, v3, v9
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v5
; GFX9-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
; GFX9-NEXT: v_sub_u32_e32 v4, 64, v16
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX9-NEXT: v_or_b32_e32 v2, v21, v10
+; GFX9-NEXT: v_or_b32_e32 v3, v22, v11
; GFX9-NEXT: v_lshrrev_b64 v[4:5], v4, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index fba617616b374..a7bb61abeb3d3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -1920,268 +1920,268 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_srem_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v10, 0x1000
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[10:11], v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v5, v7, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v4, v7
-; GISEL-NEXT: v_xor_b32_e32 v7, v8, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v5
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
-; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v5
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v7, vcc
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v8
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v4
-; GISEL-NEXT: v_trunc_f32_e32 v8, v8
-; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v8
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v4, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v4, v8
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
-; GISEL-NEXT: v_mul_lo_u32 v13, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v7, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GISEL-NEXT: v_lshl_b64 v[9:10], v[7:8], v4
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[7:8], v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v11, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v7, v1, v6
+; GISEL-NEXT: v_xor_b32_e32 v0, v9, v11
+; GISEL-NEXT: v_xor_b32_e32 v1, v10, v11
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v0
+; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v1
+; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v0
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mac_f32_e32 v9, 0x4f800000, v10
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v9, v9
+; GISEL-NEXT: v_mul_f32_e32 v9, 0x5f7ffffc, v9
+; GISEL-NEXT: v_mul_f32_e32 v10, 0x2f800000, v9
+; GISEL-NEXT: v_trunc_f32_e32 v10, v10
+; GISEL-NEXT: v_mac_f32_e32 v9, 0xcf800000, v10
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v10
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v16, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v18, v15, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v16, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
-; GISEL-NEXT: v_mul_lo_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; GISEL-NEXT: v_mul_hi_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v9
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v19, 0
-; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v19, v8
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v19, v[12:13]
-; GISEL-NEXT: v_xor_b32_e32 v15, v0, v4
-; GISEL-NEXT: v_mul_lo_u32 v0, v16, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v14
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], v6
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v9
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v16, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v18, v15, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v16, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v16, v10, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v8, v9
+; GISEL-NEXT: v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT: v_mul_lo_u32 v13, v7, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v16, v8, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v15, v8
-; GISEL-NEXT: v_xor_b32_e32 v14, v1, v4
-; GISEL-NEXT: v_mul_lo_u32 v1, v14, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v8, v10
+; GISEL-NEXT: v_mul_lo_u32 v13, v7, v10
+; GISEL-NEXT: v_mul_hi_u32 v10, v7, v10
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v8
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_mul_lo_u32 v12, v14, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v0, v1
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v8
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v16, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v8, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[1:2]
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v7, v16, v[8:9]
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v15, v0
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v15, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v15, v7
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v13, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v1, v6, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v0, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v11
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v1
-; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], v11, v1, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
-; GISEL-NEXT: v_xor_b32_e32 v8, v8, v1
-; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v8
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v16, v7
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v9
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v10, v9
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v0, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v0, v13, v[10:11]
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v1, v15, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v7, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v1
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v5
-; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v7
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v0, v7, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v9, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v9
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v9
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v20, v[1:2]
-; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
-; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v5
-; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v7, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v14, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[9:10]
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc
-; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_mul_lo_u32 v9, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v23, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT: v_sub_i32_e64 v11, s[4:5], v8, v0
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v1, vcc
+; GISEL-NEXT: v_subbrev_u32_e64 v12, vcc, 0, v7, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v12, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v12, v1
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v14, vcc
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], v7, v1, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT: v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v4, v7
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v5, v7, vcc
+; GISEL-NEXT: v_xor_b32_e32 v5, v2, v6
+; GISEL-NEXT: v_xor_b32_e32 v4, v3, v6
+; GISEL-NEXT: v_xor_b32_e32 v3, v8, v7
+; GISEL-NEXT: v_xor_b32_e32 v2, v9, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v2
+; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v3
+; GISEL-NEXT: v_subb_u32_e32 v16, vcc, 0, v2, vcc
+; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v8
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v7, v7
+; GISEL-NEXT: v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v7
+; GISEL-NEXT: v_trunc_f32_e32 v8, v8
+; GISEL-NEXT: v_mac_f32_e32 v7, 0xcf800000, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v8
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v14, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v13, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v20, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v13, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v15, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v16, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v7, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v19, v14, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v14, v0
-; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
-; GISEL-NEXT: v_mul_lo_u32 v2, v16, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v13, v11
; GISEL-NEXT: v_mul_lo_u32 v10, v14, v11
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
-; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; GISEL-NEXT: v_mul_hi_u32 v9, v14, v11
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_mul_lo_u32 v10, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v7
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v8, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v14, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v13, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v13, v11
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v16, v2, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v10, v12, v2
-; GISEL-NEXT: v_xor_b32_e32 v11, v3, v7
-; GISEL-NEXT: v_mul_lo_u32 v3, v11, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v11, v0
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v13, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v14, v8, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v5, v7
+; GISEL-NEXT: v_mul_lo_u32 v10, v5, v8
+; GISEL-NEXT: v_mul_lo_u32 v11, v4, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v4, v7
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v12, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v10, v3
-; GISEL-NEXT: v_mul_lo_u32 v10, v11, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_mul_hi_u32 v10, v5, v8
+; GISEL-NEXT: v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v4, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v10, v11, v2
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v13, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v11, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v11, v3
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v8, v7
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v11, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v2, v13, v[9:10]
+; GISEL-NEXT: v_subb_u32_e64 v7, s[4:5], v4, v11, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v4, v11
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v3
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v9, vcc, v2, v6
-; GISEL-NEXT: v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v6
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v10, v8
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[4:5]
+; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v5, v3
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v2, vcc
+; GISEL-NEXT: v_subbrev_u32_e64 v10, vcc, 0, v4, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v10, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v9, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v10, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v12, vcc
+; GISEL-NEXT: v_sub_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT: v_subb_u32_e64 v2, s[4:5], v4, v2, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v6
+; GISEL-NEXT: v_xor_b32_e32 v4, v2, v6
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v3, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v4, v6, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64_pow2_shl_denom:
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 4652d0ae60fd0..19fe3152c2f3e 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -1556,21 +1556,21 @@ define <16 x ptr> @addrspacecast_v16p6_to_v16p0(<16 x ptr addrspace(6)> %ptr) {
; HSA-NEXT: v_mov_b32_e32 v24, v12
; HSA-NEXT: v_mov_b32_e32 v20, v10
; HSA-NEXT: v_mov_b32_e32 v16, v8
-; HSA-NEXT: v_mov_b32_e32 v14, v7
; HSA-NEXT: v_mov_b32_e32 v12, v6
-; HSA-NEXT: v_mov_b32_e32 v10, v5
; HSA-NEXT: v_mov_b32_e32 v8, v4
-; HSA-NEXT: v_mov_b32_e32 v6, v3
; HSA-NEXT: v_mov_b32_e32 v4, v2
; HSA-NEXT: v_mov_b32_e32 v2, v1
; HSA-NEXT: v_mov_b32_e32 v1, 0
-; HSA-NEXT: v_mov_b32_e32 v3, 0
-; HSA-NEXT: v_mov_b32_e32 v5, 0
-; HSA-NEXT: v_mov_b32_e32 v7, 0
+; HSA-NEXT: v_mov_b32_e32 v6, v3
+; HSA-NEXT: v_mov_b32_e32 v10, v5
+; HSA-NEXT: v_mov_b32_e32 v14, v7
; HSA-NEXT: v_mov_b32_e32 v18, v9
; HSA-NEXT: v_mov_b32_e32 v22, v11
; HSA-NEXT: v_mov_b32_e32 v26, v13
; HSA-NEXT: v_mov_b32_e32 v30, v15
+; HSA-NEXT: v_mov_b32_e32 v3, 0
+; HSA-NEXT: v_mov_b32_e32 v5, 0
+; HSA-NEXT: v_mov_b32_e32 v7, 0
; HSA-NEXT: v_mov_b32_e32 v9, 0
; HSA-NEXT: v_mov_b32_e32 v11, 0
; HSA-NEXT: v_mov_b32_e32 v13, 0
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 727728145dfe0..e89ed5b0d82a6 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -370,76 +370,151 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX908-LABEL: v32_asm_def_use:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v33, v1
-; GFX908-NEXT: v_mov_b32_e32 v34, v0
+; GFX908-NEXT: v_mov_b32_e32 v32, v1
+; GFX908-NEXT: v_mov_b32_e32 v33, v0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def v[0:31] a[0:15]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a15
-; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def v32
-; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a15
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a14
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a13
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a12
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a11
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a14
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a13
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a12
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a11
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a10
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a9
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a8
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a7
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX908-NEXT: s_nop 9
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a6
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a5
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a4
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a3
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v35
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX908-NEXT: ;;#ASMSTART
+; GFX908-NEXT: ; def v32
+; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
-; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_write_b32 a32, v35
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: s_nop 7
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a3 v[0:31]
; GFX908-NEXT: ;;#ASMEND
@@ -451,9 +526,8 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-LABEL: v32_asm_def_use:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a32 ; Reload Reuse
-; GFX90A-NEXT: v_mov_b32_e32 v34, v0
-; GFX90A-NEXT: v_mov_b32_e32 v33, v1
+; GFX90A-NEXT: v_mov_b32_e32 v33, v0
+; GFX90A-NEXT: v_mov_b32_e32 v32, v1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:31] a[0:15]
; GFX90A-NEXT: ;;#ASMEND
@@ -473,20 +547,53 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-NEXT: v_accvgpr_mov_b32 a18, a2
; GFX90A-NEXT: v_accvgpr_mov_b32 a17, a1
; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a0
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v32
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
+; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a1
+; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v35 ; Reload Reuse
-; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
@@ -1077,55 +1184,118 @@ define void @no_free_vgprs_at_sgpr_to_agpr_copy(float %v0, float %v1) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2
; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1
; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0
-; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX90A-NEXT: v_accvgpr_read_b32 v39, a11 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v38, a12 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v37, a13 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v36, a14 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a15 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX90A-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX90A-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX90A-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX90A-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX90A-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX90A-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX90A-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX90A-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX90A-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX90A-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX90A-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX90A-NEXT: v_accvgpr_read_b32 v15, a15
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX90A-NEXT: v_accvgpr_write_b32 a11, v39 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a12, v38 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a13, v37 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a14, v36 ; Reload Reuse
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a15, v35 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
+; GFX90A-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX90A-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX90A-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX90A-NEXT: v_accvgpr_write_b32 a5, v5
+; GFX90A-NEXT: v_accvgpr_write_b32 a6, v6
+; GFX90A-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX90A-NEXT: v_accvgpr_write_b32 a8, v8
+; GFX90A-NEXT: v_accvgpr_write_b32 a9, v9
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v10
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v12
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v13
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v14
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v15
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
+; GFX90A-NEXT: v_accvgpr_write_b32 a32, v34 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v34 ; Reload Reuse
; GFX90A-NEXT: s_setpc_b64 s[30:31]
%asm = call { <32 x i32>, <16 x float> } asm sideeffect "; def $0 $1","=${v[0:31]},=${s[0:15]}"()
%vgpr0 = extractvalue { <32 x i32>, <16 x float> } %asm, 0
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2d5fc5cf22425..47ad460609461 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -12,10 +12,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -62,10 +62,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -112,10 +112,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -615,10 +615,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -665,10 +665,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -715,10 +715,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1426,10 +1426,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1476,10 +1476,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1526,10 +1526,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2029,10 +2029,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2079,10 +2079,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2129,10 +2129,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2640,10 +2640,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2690,10 +2690,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2740,10 +2740,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -3243,16 +3243,15 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB10_2
; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3271,22 +3270,20 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; SI-NEXT: .LBB10_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; VI-NEXT: s_cbranch_execz .LBB10_2
; VI-NEXT: ; %bb.1: ; %cmp.true
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3305,22 +3302,20 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; VI-NEXT: .LBB10_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB10_2
; GFX9-NEXT: ; %bb.1: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3339,7 +3334,6 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; GFX9-NEXT: .LBB10_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v32i32:
@@ -22226,10 +22220,10 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -22276,10 +22270,10 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -30263,10 +30257,10 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -30313,10 +30307,10 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -31898,10 +31892,10 @@ define <32 x i32> @bitcast_v64f16_to_v32i32(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -33441,10 +33435,10 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -33491,10 +33485,10 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -34907,10 +34901,10 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36008,10 +36002,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36058,10 +36052,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36108,10 +36102,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36819,10 +36813,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36869,10 +36863,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36919,10 +36913,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37430,10 +37424,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37480,10 +37474,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37530,10 +37524,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -38241,16 +38235,15 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB34_2
; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38269,22 +38262,20 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; SI-NEXT: .LBB34_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; VI-NEXT: s_cbranch_execz .LBB34_2
; VI-NEXT: ; %bb.1: ; %cmp.true
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38303,22 +38294,20 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; VI-NEXT: .LBB34_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB34_2
; GFX9-NEXT: ; %bb.1: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38337,7 +38326,6 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; GFX9-NEXT: .LBB34_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v32f32:
@@ -58114,10 +58102,10 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -58164,10 +58152,10 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -66342,10 +66330,10 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -66392,10 +66380,10 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -68124,10 +68112,10 @@ define <32 x float> @bitcast_v64f16_to_v32f32(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -69667,10 +69655,10 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -69717,10 +69705,10 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -71280,10 +71268,10 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72381,10 +72369,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72431,10 +72419,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72481,10 +72469,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72991,10 +72979,10 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73015,20 +73003,18 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: .LBB54_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73049,20 +73035,18 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: .LBB54_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73083,11 +73067,9 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: .LBB54_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v16i64:
@@ -92004,10 +91986,10 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -92054,10 +92036,10 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -100017,10 +99999,10 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -100067,10 +100049,10 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -101660,10 +101642,10 @@ define <16 x i64> @bitcast_v64f16_to_v16i64(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -103203,10 +103185,10 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -103253,10 +103235,10 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -104677,10 +104659,10 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -124925,10 +124907,10 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -124942,7 +124924,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -124953,16 +124934,15 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB76_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -124976,7 +124956,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -124987,7 +124966,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB76_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64bf16:
@@ -133050,10 +133028,10 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -133067,7 +133045,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -133078,16 +133055,15 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB80_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -133101,7 +133077,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -133112,7 +133087,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB80_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64f16:
@@ -134737,10 +134711,10 @@ define <16 x double> @bitcast_v64f16_to_v16f64(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136265,10 +136239,10 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136282,7 +136256,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -136293,16 +136266,15 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB84_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136316,7 +136288,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -136327,7 +136298,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB84_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64i16:
@@ -137783,10 +137753,10 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -216096,10 +216066,10 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -225741,10 +225711,10 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -227726,10 +227696,10 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -230074,10 +230044,10 @@ define <64 x half> @bitcast_v64i16_to_v64f16(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 224b73353c3bc..a94117fee01a0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -12778,10 +12778,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[0:1], v[38:39], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[48:49], 8
; SI-NEXT: v_lshr_b64 v[19:20], v[36:37], 24
-; SI-NEXT: v_lshr_b64 v[32:33], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[36:37], 16
; SI-NEXT: v_lshr_b64 v[17:18], v[36:37], 8
; SI-NEXT: v_lshr_b64 v[11:12], v[34:35], 24
-; SI-NEXT: v_lshr_b64 v[28:29], v[34:35], 16
+; SI-NEXT: v_lshr_b64 v[32:33], v[34:35], 16
; SI-NEXT: v_lshr_b64 v[9:10], v[34:35], 8
; SI-NEXT: v_lshr_b64 v[3:4], v[38:39], 24
; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 8
@@ -12849,10 +12849,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v24, s42
; SI-NEXT: v_mov_b32_e32 v25, s44
; SI-NEXT: v_mov_b32_e32 v19, s24
-; SI-NEXT: v_mov_b32_e32 v32, s26
+; SI-NEXT: v_mov_b32_e32 v28, s26
; SI-NEXT: v_mov_b32_e32 v17, s28
; SI-NEXT: v_mov_b32_e32 v11, s10
-; SI-NEXT: v_mov_b32_e32 v28, s12
+; SI-NEXT: v_mov_b32_e32 v32, s12
; SI-NEXT: v_mov_b32_e32 v9, s14
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v0, s6
@@ -12861,10 +12861,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v38
; SI-NEXT: v_mov_b32_e32 v4, v39
-; SI-NEXT: v_mov_b32_e32 v10, v28
+; SI-NEXT: v_mov_b32_e32 v10, v32
; SI-NEXT: v_mov_b32_e32 v8, v34
; SI-NEXT: v_mov_b32_e32 v12, v35
-; SI-NEXT: v_mov_b32_e32 v18, v32
+; SI-NEXT: v_mov_b32_e32 v18, v28
; SI-NEXT: v_mov_b32_e32 v16, v36
; SI-NEXT: v_mov_b32_e32 v20, v37
; SI-NEXT: v_mov_b32_e32 v26, v24
@@ -25905,14 +25905,14 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_cbranch_execnz .LBB85_4
; VI-NEXT: .LBB85_2: ; %cmp.true
+; VI-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; VI-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
-; VI-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
-; VI-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; VI-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
+; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
-; VI-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
; VI-NEXT: v_lshrrev_b32_e32 v31, 24, v25
; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
@@ -26036,14 +26036,14 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_cbranch_execnz .LBB85_4
; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; GFX9-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
-; GFX9-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
-; GFX9-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; GFX9-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
+; GFX9-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX9-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
-; GFX9-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 24, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 8, v25
@@ -37190,88 +37190,88 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: .LBB105_2: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s79
; SI-NEXT: v_cvt_f32_f16_e32 v1, s22
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s78
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s20
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
; SI-NEXT: v_or_b32_e32 v33, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s78
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s23
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s77
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
-; SI-NEXT: v_bfe_u32 v31, v30, 8, 8
-; SI-NEXT: v_or_b32_e32 v34, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s77
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s20
-; SI-NEXT: v_lshr_b64 v[24:25], v[33:34], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[33:34], 24
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[25:26], v[33:34], 8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v34
-; SI-NEXT: v_or_b32_e32 v35, v1, v0
+; SI-NEXT: v_or_b32_e32 v34, v2, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s76
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v35, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s21
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s75
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v22, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_bfe_u32 v23, v22, 8, 8
-; SI-NEXT: v_or_b32_e32 v36, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s75
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s18
-; SI-NEXT: v_lshr_b64 v[19:20], v[35:36], 24
-; SI-NEXT: v_lshr_b64 v[20:21], v[35:36], 16
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[17:18], v[35:36], 8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v36
-; SI-NEXT: v_or_b32_e32 v37, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s74
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s19
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
+; SI-NEXT: v_or_b32_e32 v36, v0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s74
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s19
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
-; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
-; SI-NEXT: v_or_b32_e32 v38, v1, v0
+; SI-NEXT: v_or_b32_e32 v37, v2, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s73
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_or_b32_e32 v38, v1, v2
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_lshr_b64 v[11:12], v[37:38], 24
-; SI-NEXT: v_lshr_b64 v[12:13], v[37:38], 16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s72
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[9:10], v[37:38], 8
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
; SI-NEXT: v_or_b32_e32 v48, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s72
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
-; SI-NEXT: v_or_b32_e32 v49, v1, v0
+; SI-NEXT: v_or_b32_e32 v49, v2, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[37:38], 24
+; SI-NEXT: v_lshr_b64 v[16:17], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[33:34], 16
; SI-NEXT: v_lshr_b64 v[3:4], v[48:49], 24
; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 8
+; SI-NEXT: v_lshr_b64 v[12:13], v[37:38], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[37:38], 8
+; SI-NEXT: v_lshr_b64 v[19:20], v[35:36], 24
+; SI-NEXT: v_lshr_b64 v[17:18], v[35:36], 8
+; SI-NEXT: v_lshr_b64 v[27:28], v[33:34], 24
+; SI-NEXT: v_lshr_b64 v[25:26], v[33:34], 8
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v49
+; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
+; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v36
+; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v34
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
+; SI-NEXT: v_bfe_u32 v23, v22, 8, 8
+; SI-NEXT: v_bfe_u32 v31, v30, 8, 8
; SI-NEXT: s_branch .LBB105_5
; SI-NEXT: .LBB105_3:
; SI-NEXT: ; implicit-def: $sgpr10
@@ -37327,7 +37327,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s14
; SI-NEXT: v_mov_b32_e32 v9, s24
; SI-NEXT: v_mov_b32_e32 v19, s28
-; SI-NEXT: v_mov_b32_e32 v20, s40
+; SI-NEXT: v_mov_b32_e32 v16, s40
; SI-NEXT: v_mov_b32_e32 v17, s42
; SI-NEXT: v_mov_b32_e32 v27, s46
; SI-NEXT: v_mov_b32_e32 v24, s56
@@ -37339,7 +37339,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, v12
; SI-NEXT: v_mov_b32_e32 v8, v37
; SI-NEXT: v_mov_b32_e32 v12, v38
-; SI-NEXT: v_mov_b32_e32 v18, v20
+; SI-NEXT: v_mov_b32_e32 v18, v16
; SI-NEXT: v_mov_b32_e32 v16, v35
; SI-NEXT: v_mov_b32_e32 v20, v36
; SI-NEXT: v_mov_b32_e32 v26, v24
@@ -41239,38 +41239,38 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_or_b32_e32 v6, 0x400000, v2
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_lshrrev_b64 v[24:25], 16, v[5:6]
+; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v25, v28
+; VI-NEXT: v_lshrrev_b64 v[24:25], 16, v[5:6]
; VI-NEXT: v_mov_b32_e32 v1, v4
; VI-NEXT: v_mov_b32_e32 v9, v12
; VI-NEXT: v_mov_b32_e32 v17, v20
-; VI-NEXT: v_lshrrev_b64 v[36:37], 24, v[24:25]
-; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[16:17]
-; VI-NEXT: v_lshrrev_b64 v[34:35], 24, v[8:9]
-; VI-NEXT: v_lshrrev_b64 v[32:33], 24, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v25, v28
+; VI-NEXT: v_lshrrev_b64 v[50:51], 24, v[24:25]
+; VI-NEXT: v_lshrrev_b64 v[48:49], 24, v[16:17]
+; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[8:9]
+; VI-NEXT: v_lshrrev_b64 v[34:35], 24, v[0:1]
; VI-NEXT: v_lshrrev_b32_e32 v31, 24, v28
; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v28
; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v28
; VI-NEXT: v_lshrrev_b32_e32 v26, 16, v24
-; VI-NEXT: v_lshrrev_b32_e32 v25, 8, v24
+; VI-NEXT: v_lshrrev_b32_e32 v32, 8, v24
; VI-NEXT: v_lshrrev_b32_e32 v23, 24, v20
; VI-NEXT: v_lshrrev_b32_e32 v22, 16, v20
; VI-NEXT: v_lshrrev_b32_e32 v21, 8, v20
; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; VI-NEXT: v_lshrrev_b32_e32 v17, 8, v16
+; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v16
; VI-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v12
; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v12
; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; VI-NEXT: v_lshrrev_b32_e32 v36, 8, v8
; VI-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; VI-NEXT: v_lshrrev_b32_e32 v39, 8, v0
; VI-NEXT: s_branch .LBB109_5
; VI-NEXT: .LBB109_3:
; VI-NEXT: ; implicit-def: $sgpr25
@@ -41300,22 +41300,22 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: s_branch .LBB109_2
; VI-NEXT: .LBB109_4:
; VI-NEXT: v_mov_b32_e32 v26, s59
-; VI-NEXT: v_mov_b32_e32 v25, s58
+; VI-NEXT: v_mov_b32_e32 v32, s58
; VI-NEXT: v_mov_b32_e32 v31, s57
; VI-NEXT: v_mov_b32_e32 v30, s56
; VI-NEXT: v_mov_b32_e32 v29, s47
; VI-NEXT: v_mov_b32_e32 v18, s46
-; VI-NEXT: v_mov_b32_e32 v17, s45
+; VI-NEXT: v_mov_b32_e32 v33, s45
; VI-NEXT: v_mov_b32_e32 v23, s44
; VI-NEXT: v_mov_b32_e32 v22, s43
; VI-NEXT: v_mov_b32_e32 v21, s42
; VI-NEXT: v_mov_b32_e32 v10, s41
-; VI-NEXT: v_mov_b32_e32 v9, s40
+; VI-NEXT: v_mov_b32_e32 v36, s40
; VI-NEXT: v_mov_b32_e32 v15, s29
; VI-NEXT: v_mov_b32_e32 v14, s28
; VI-NEXT: v_mov_b32_e32 v13, s27
; VI-NEXT: v_mov_b32_e32 v2, s26
-; VI-NEXT: v_mov_b32_e32 v1, s25
+; VI-NEXT: v_mov_b32_e32 v39, s25
; VI-NEXT: v_mov_b32_e32 v7, s24
; VI-NEXT: v_mov_b32_e32 v6, s15
; VI-NEXT: v_mov_b32_e32 v5, s14
@@ -41327,15 +41327,19 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v12, s19
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v4, s17
-; VI-NEXT: v_mov_b32_e32 v36, s10
-; VI-NEXT: v_mov_b32_e32 v37, s8
-; VI-NEXT: v_mov_b32_e32 v34, s6
-; VI-NEXT: v_mov_b32_e32 v32, s4
+; VI-NEXT: v_mov_b32_e32 v50, s10
+; VI-NEXT: v_mov_b32_e32 v48, s8
+; VI-NEXT: v_mov_b32_e32 v37, s6
+; VI-NEXT: v_mov_b32_e32 v34, s4
; VI-NEXT: .LBB109_5: ; %end
-; VI-NEXT: v_mov_b32_e32 v3, v32
-; VI-NEXT: v_mov_b32_e32 v11, v34
-; VI-NEXT: v_mov_b32_e32 v19, v37
-; VI-NEXT: v_mov_b32_e32 v27, v36
+; VI-NEXT: v_mov_b32_e32 v3, v34
+; VI-NEXT: v_mov_b32_e32 v11, v37
+; VI-NEXT: v_mov_b32_e32 v19, v48
+; VI-NEXT: v_mov_b32_e32 v27, v50
+; VI-NEXT: v_mov_b32_e32 v1, v39
+; VI-NEXT: v_mov_b32_e32 v9, v36
+; VI-NEXT: v_mov_b32_e32 v17, v33
+; VI-NEXT: v_mov_b32_e32 v25, v32
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16bf16_to_v32i8_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index 69562f9d0a70d..8a8d3ae036a5c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -31054,170 +31054,170 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s94, 16
-; SI-NEXT: s_or_b32 s47, s5, s7
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s48, 16
+; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
; SI-NEXT: s_lshl_b32 s7, s95, 16
-; SI-NEXT: s_or_b32 s43, s5, s7
+; SI-NEXT: s_or_b32 s42, s4, s56
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s58, s49, 16
+; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
; SI-NEXT: s_lshl_b32 s7, s30, 16
-; SI-NEXT: s_or_b32 s41, s5, s7
+; SI-NEXT: s_or_b32 s40, s4, s58
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s60, s50, 16
+; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s31, 16
-; SI-NEXT: s_or_b32 s15, s5, s7
+; SI-NEXT: s_or_b32 s14, s4, s60
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s62, s51, 16
+; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
; SI-NEXT: s_lshl_b32 s7, s34, 16
-; SI-NEXT: s_or_b32 s13, s5, s7
+; SI-NEXT: s_or_b32 s12, s4, s62
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s72, s52, 16
+; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
; SI-NEXT: s_lshl_b32 s7, s35, 16
-; SI-NEXT: s_or_b32 s11, s5, s7
+; SI-NEXT: s_or_b32 s10, s4, s72
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s74, s53, 16
+; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s36, 16
-; SI-NEXT: s_or_b32 s9, s5, s7
+; SI-NEXT: s_or_b32 s8, s4, s74
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s76, s54, 16
+; SI-NEXT: s_or_b32 s77, s5, s7
; SI-NEXT: s_and_b32 s5, s55, 0xffff
; SI-NEXT: s_lshl_b32 s7, s37, 16
-; SI-NEXT: s_or_b32 s7, s5, s7
+; SI-NEXT: s_or_b32 s6, s4, s76
+; SI-NEXT: s_and_b32 s4, s65, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s64, 16
+; SI-NEXT: s_or_b32 s47, s5, s7
; SI-NEXT: s_and_b32 s5, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s45, s38, 16
-; SI-NEXT: s_and_b32 s44, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s48, 16
-; SI-NEXT: s_or_b32 s5, s5, s45
-; SI-NEXT: s_and_b32 s45, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s39, 16
-; SI-NEXT: s_lshl_b32 s42, s49, 16
-; SI-NEXT: s_or_b32 s91, s45, s56
-; SI-NEXT: s_or_b32 s44, s44, s46
-; SI-NEXT: s_lshr_b64 s[56:57], s[46:47], 16
-; SI-NEXT: s_and_b32 s46, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s50, 16
-; SI-NEXT: s_or_b32 s46, s46, s42
-; SI-NEXT: s_lshr_b64 s[58:59], s[42:43], 16
-; SI-NEXT: s_and_b32 s42, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s51, 16
-; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_lshr_b64 s[60:61], s[40:41], 16
-; SI-NEXT: s_and_b32 s40, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s52, 16
-; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_lshr_b64 s[62:63], s[14:15], 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s53, 16
-; SI-NEXT: s_or_b32 s14, s14, s12
-; SI-NEXT: s_lshr_b64 s[72:73], s[12:13], 16
-; SI-NEXT: s_and_b32 s12, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s54, 16
-; SI-NEXT: s_or_b32 s12, s12, s10
-; SI-NEXT: s_lshr_b64 s[74:75], s[10:11], 16
-; SI-NEXT: s_and_b32 s10, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s64, 16
-; SI-NEXT: s_or_b32 s10, s10, s8
-; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
-; SI-NEXT: s_and_b32 s8, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s4, s67, 16
-; SI-NEXT: s_or_b32 s8, s8, s6
-; SI-NEXT: s_lshr_b64 s[78:79], s[6:7], 16
-; SI-NEXT: s_and_b32 s6, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s38, 16
+; SI-NEXT: s_or_b32 s4, s4, s46
+; SI-NEXT: s_lshl_b32 s44, s67, 16
+; SI-NEXT: s_or_b32 s45, s5, s7
+; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s39, 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[46:47], 16
+; SI-NEXT: s_and_b32 s46, s68, 0xffff
+; SI-NEXT: s_or_b32 s91, s5, s7
; SI-NEXT: s_lshl_b32 s90, s70, 16
-; SI-NEXT: s_or_b32 s6, s6, s4
-; SI-NEXT: s_lshr_b64 s[88:89], s[4:5], 16
-; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_mov_b32 s45, s47
-; SI-NEXT: s_mov_b32 s47, s43
-; SI-NEXT: s_mov_b32 s43, s41
-; SI-NEXT: s_mov_b32 s41, s15
-; SI-NEXT: s_mov_b32 s15, s13
-; SI-NEXT: s_mov_b32 s13, s11
-; SI-NEXT: s_mov_b32 s11, s9
-; SI-NEXT: s_mov_b32 s9, s7
-; SI-NEXT: s_mov_b32 s7, s5
-; SI-NEXT: s_or_b32 s4, s4, s90
-; SI-NEXT: s_mov_b32 s5, s91
+; SI-NEXT: s_or_b32 s46, s46, s44
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_and_b32 s44, s71, 0xffff
+; SI-NEXT: s_mov_b32 s43, s57
+; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
+; SI-NEXT: s_mov_b32 s41, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
+; SI-NEXT: s_mov_b32 s15, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_mov_b32 s13, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
+; SI-NEXT: s_mov_b32 s11, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
+; SI-NEXT: s_mov_b32 s9, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
+; SI-NEXT: s_mov_b32 s7, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[76:77], 16
+; SI-NEXT: s_mov_b32 s5, s47
+; SI-NEXT: s_mov_b32 s47, s45
+; SI-NEXT: s_or_b32 s44, s44, s90
+; SI-NEXT: s_mov_b32 s45, s91
; SI-NEXT: s_lshr_b64 s[90:91], s[90:91], 16
; SI-NEXT: s_cbranch_execnz .LBB57_3
; SI-NEXT: .LBB57_2: ; %cmp.true
; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_and_b32 s4, s71, 0xffff
; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s44, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s39, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s39, 16
; SI-NEXT: s_add_i32 s68, s68, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s67, 16
+; SI-NEXT: s_add_i32 s45, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s38, 16
+; SI-NEXT: s_add_i32 s46, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s38, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s65, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s64, 16
+; SI-NEXT: s_add_i32 s47, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s65, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s64, 16
; SI-NEXT: s_add_i32 s55, s55, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s55, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s37, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s37, 16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s54, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s54, 16
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s36, 16
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s36, 16
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s53, 16
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s53, 16
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s35, 16
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s35, 16
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s52, 16
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s10, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s52, 16
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_and_b32 s15, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s34, 16
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s34, 16
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s15, s24, s15
-; SI-NEXT: s_and_b32 s22, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s51, 16
-; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s12, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s51, 16
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s40, s22, 0x30000
-; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s31, 16
-; SI-NEXT: s_or_b32 s22, s23, s22
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_and_b32 s13, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s31, 16
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_add_i32 s41, s22, 0x30000
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s50, 16
-; SI-NEXT: s_or_b32 s20, s22, s20
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_and_b32 s14, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s50, 16
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s42, s20, 0x30000
-; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s30, 16
-; SI-NEXT: s_or_b32 s20, s21, s20
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_and_b32 s15, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s30, 16
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_add_i32 s43, s20, 0x30000
+; SI-NEXT: s_or_b32 s15, s20, s15
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_lshl_b32 s20, s49, 16
; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s46, s18, 0x30000
+; SI-NEXT: s_add_i32 s40, s18, 0x30000
; SI-NEXT: s_and_b32 s18, s19, 0xffff
; SI-NEXT: s_lshl_b32 s19, s95, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s47, s18, 0x30000
+; SI-NEXT: s_add_i32 s41, s18, 0x30000
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_lshl_b32 s18, s48, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s44, s16, 0x30000
+; SI-NEXT: s_add_i32 s42, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s17, 0xffff
; SI-NEXT: s_lshl_b32 s17, s94, 16
; SI-NEXT: s_or_b32 s16, s17, s16
@@ -31233,109 +31233,109 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: s_add_i32 s45, s16, 0x30000
-; SI-NEXT: s_lshr_b64 s[56:57], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[58:59], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[60:61], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[72:73], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[88:89], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[4:5], 16
-; SI-NEXT: s_lshr_b32 s94, s45, 16
-; SI-NEXT: s_lshr_b32 s95, s47, 16
-; SI-NEXT: s_lshr_b32 s30, s43, 16
-; SI-NEXT: s_lshr_b32 s31, s41, 16
-; SI-NEXT: s_lshr_b32 s34, s15, 16
-; SI-NEXT: s_lshr_b32 s35, s13, 16
-; SI-NEXT: s_lshr_b32 s36, s11, 16
-; SI-NEXT: s_lshr_b32 s37, s9, 16
-; SI-NEXT: s_lshr_b32 s38, s7, 16
-; SI-NEXT: s_lshr_b32 s39, s5, 16
+; SI-NEXT: s_add_i32 s43, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[56:57], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[44:45], 16
+; SI-NEXT: s_lshr_b32 s94, s43, 16
+; SI-NEXT: s_lshr_b32 s95, s41, 16
+; SI-NEXT: s_lshr_b32 s30, s15, 16
+; SI-NEXT: s_lshr_b32 s31, s13, 16
+; SI-NEXT: s_lshr_b32 s34, s11, 16
+; SI-NEXT: s_lshr_b32 s35, s9, 16
+; SI-NEXT: s_lshr_b32 s36, s7, 16
+; SI-NEXT: s_lshr_b32 s37, s5, 16
+; SI-NEXT: s_lshr_b32 s38, s47, 16
+; SI-NEXT: s_lshr_b32 s39, s45, 16
; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: s_and_b32 s16, s44, 0xffff
+; SI-NEXT: s_and_b32 s16, s42, 0xffff
; SI-NEXT: s_lshl_b32 s17, s56, 16
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s45, 0xffff
+; SI-NEXT: s_and_b32 s17, s43, 0xffff
; SI-NEXT: s_lshl_b32 s18, s94, 16
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_and_b32 s18, s46, 0xffff
+; SI-NEXT: s_and_b32 s18, s40, 0xffff
; SI-NEXT: s_lshl_b32 s19, s58, 16
; SI-NEXT: s_or_b32 s18, s18, s19
-; SI-NEXT: s_and_b32 s19, s47, 0xffff
+; SI-NEXT: s_and_b32 s19, s41, 0xffff
; SI-NEXT: s_lshl_b32 s20, s95, 16
; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: s_and_b32 s20, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s60, 16
-; SI-NEXT: s_or_b32 s20, s20, s21
-; SI-NEXT: s_and_b32 s21, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s30, 16
-; SI-NEXT: s_or_b32 s21, s21, s22
-; SI-NEXT: s_and_b32 s22, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s62, 16
-; SI-NEXT: s_or_b32 s22, s22, s23
-; SI-NEXT: s_and_b32 s23, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s31, 16
-; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s72, 16
-; SI-NEXT: s_or_b32 s14, s14, s24
+; SI-NEXT: s_lshl_b32 s20, s60, 16
+; SI-NEXT: s_or_b32 s14, s14, s20
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s34, 16
-; SI-NEXT: s_or_b32 s15, s15, s24
+; SI-NEXT: s_lshl_b32 s20, s30, 16
+; SI-NEXT: s_or_b32 s15, s15, s20
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s74, 16
-; SI-NEXT: s_or_b32 s12, s12, s24
+; SI-NEXT: s_lshl_b32 s20, s62, 16
+; SI-NEXT: s_or_b32 s12, s12, s20
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s35, 16
-; SI-NEXT: s_or_b32 s13, s13, s24
+; SI-NEXT: s_lshl_b32 s20, s31, 16
+; SI-NEXT: s_or_b32 s13, s13, s20
; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s76, 16
-; SI-NEXT: s_or_b32 s10, s10, s24
+; SI-NEXT: s_lshl_b32 s20, s72, 16
+; SI-NEXT: s_or_b32 s10, s10, s20
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s36, 16
-; SI-NEXT: s_or_b32 s11, s11, s24
+; SI-NEXT: s_lshl_b32 s20, s34, 16
+; SI-NEXT: s_or_b32 s11, s11, s20
; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s78, 16
-; SI-NEXT: s_or_b32 s8, s8, s24
+; SI-NEXT: s_lshl_b32 s20, s74, 16
+; SI-NEXT: s_or_b32 s8, s8, s20
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s37, 16
-; SI-NEXT: s_or_b32 s9, s9, s24
+; SI-NEXT: s_lshl_b32 s20, s35, 16
+; SI-NEXT: s_or_b32 s9, s9, s20
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s88, 16
-; SI-NEXT: s_or_b32 s6, s6, s24
+; SI-NEXT: s_lshl_b32 s20, s76, 16
+; SI-NEXT: s_or_b32 s6, s6, s20
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s38, 16
-; SI-NEXT: s_or_b32 s7, s7, s24
+; SI-NEXT: s_lshl_b32 s20, s36, 16
+; SI-NEXT: s_or_b32 s7, s7, s20
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s90, 16
-; SI-NEXT: s_or_b32 s4, s4, s24
+; SI-NEXT: s_lshl_b32 s20, s78, 16
+; SI-NEXT: s_or_b32 s4, s4, s20
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s37, 16
+; SI-NEXT: s_or_b32 s5, s5, s20
+; SI-NEXT: s_and_b32 s20, s46, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s88, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_and_b32 s21, s47, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s38, 16
+; SI-NEXT: s_or_b32 s21, s21, s22
+; SI-NEXT: s_and_b32 s22, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: s_or_b32 s22, s22, s23
+; SI-NEXT: s_and_b32 s23, s45, 0xffff
; SI-NEXT: s_lshl_b32 s24, s39, 16
-; SI-NEXT: s_or_b32 s5, s5, s24
+; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: v_readlane_b32 s30, v20, 22
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
-; SI-NEXT: v_mov_b32_e32 v4, s20
-; SI-NEXT: v_mov_b32_e32 v5, s21
-; SI-NEXT: v_mov_b32_e32 v6, s22
-; SI-NEXT: v_mov_b32_e32 v7, s23
-; SI-NEXT: v_mov_b32_e32 v8, s14
-; SI-NEXT: v_mov_b32_e32 v9, s15
-; SI-NEXT: v_mov_b32_e32 v10, s12
-; SI-NEXT: v_mov_b32_e32 v11, s13
-; SI-NEXT: v_mov_b32_e32 v12, s10
-; SI-NEXT: v_mov_b32_e32 v13, s11
-; SI-NEXT: v_mov_b32_e32 v14, s8
-; SI-NEXT: v_mov_b32_e32 v15, s9
-; SI-NEXT: v_mov_b32_e32 v16, s6
-; SI-NEXT: v_mov_b32_e32 v17, s7
-; SI-NEXT: v_mov_b32_e32 v18, s4
-; SI-NEXT: v_mov_b32_e32 v19, s5
+; SI-NEXT: v_mov_b32_e32 v4, s14
+; SI-NEXT: v_mov_b32_e32 v5, s15
+; SI-NEXT: v_mov_b32_e32 v6, s12
+; SI-NEXT: v_mov_b32_e32 v7, s13
+; SI-NEXT: v_mov_b32_e32 v8, s10
+; SI-NEXT: v_mov_b32_e32 v9, s11
+; SI-NEXT: v_mov_b32_e32 v10, s8
+; SI-NEXT: v_mov_b32_e32 v11, s9
+; SI-NEXT: v_mov_b32_e32 v12, s6
+; SI-NEXT: v_mov_b32_e32 v13, s7
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_mov_b32_e32 v16, s20
+; SI-NEXT: v_mov_b32_e32 v17, s21
+; SI-NEXT: v_mov_b32_e32 v18, s22
+; SI-NEXT: v_mov_b32_e32 v19, s23
; SI-NEXT: v_readlane_b32 s31, v20, 23
; SI-NEXT: v_readlane_b32 s71, v20, 21
; SI-NEXT: v_readlane_b32 s70, v20, 20
@@ -31365,25 +31365,25 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr90
; SI-NEXT: s_branch .LBB57_2
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index df100725d890c..debe8a1aff295 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -33869,232 +33869,232 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v22, s83, 25
; SI-NEXT: v_writelane_b32 v22, s84, 26
; SI-NEXT: v_writelane_b32 v22, s85, 27
-; SI-NEXT: v_writelane_b32 v22, s30, 28
-; SI-NEXT: v_writelane_b32 v22, s31, 29
-; SI-NEXT: v_readfirstlane_b32 s83, v7
-; SI-NEXT: v_readfirstlane_b32 s85, v6
-; SI-NEXT: v_readfirstlane_b32 s80, v5
-; SI-NEXT: v_readfirstlane_b32 s82, v4
-; SI-NEXT: v_readfirstlane_b32 s69, v3
-; SI-NEXT: v_readfirstlane_b32 s71, v2
-; SI-NEXT: v_readfirstlane_b32 s66, v1
-; SI-NEXT: v_readfirstlane_b32 s68, v0
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s65, s28, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s64, s26, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s55, s24, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s54, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s53, s20, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s52, s18, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
-; SI-NEXT: s_lshr_b32 s51, s16, 16
-; SI-NEXT: s_lshr_b32 s50, s83, 16
-; SI-NEXT: s_lshr_b32 s84, s85, 16
-; SI-NEXT: s_lshr_b32 s49, s80, 16
-; SI-NEXT: s_lshr_b32 s81, s82, 16
-; SI-NEXT: s_lshr_b32 s48, s69, 16
-; SI-NEXT: s_lshr_b32 s70, s71, 16
-; SI-NEXT: s_lshr_b32 s39, s66, 16
-; SI-NEXT: s_lshr_b32 s67, s68, 16
+; SI-NEXT: v_writelane_b32 v22, s86, 28
+; SI-NEXT: v_writelane_b32 v22, s87, 29
+; SI-NEXT: v_writelane_b32 v22, s30, 30
+; SI-NEXT: v_writelane_b32 v22, s31, 31
+; SI-NEXT: v_readfirstlane_b32 s85, v7
+; SI-NEXT: v_readfirstlane_b32 s87, v6
+; SI-NEXT: v_readfirstlane_b32 s82, v5
+; SI-NEXT: v_readfirstlane_b32 s84, v4
+; SI-NEXT: v_readfirstlane_b32 s71, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s68, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
+; SI-NEXT: s_lshr_b32 s48, s29, 16
+; SI-NEXT: s_lshr_b32 s67, s28, 16
+; SI-NEXT: s_lshr_b32 s39, s27, 16
+; SI-NEXT: s_lshr_b32 s66, s26, 16
+; SI-NEXT: s_lshr_b32 s38, s25, 16
+; SI-NEXT: s_lshr_b32 s65, s24, 16
+; SI-NEXT: s_lshr_b32 s37, s23, 16
+; SI-NEXT: s_lshr_b32 s64, s22, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s55, s20, 16
+; SI-NEXT: s_lshr_b32 s35, s19, 16
+; SI-NEXT: s_lshr_b32 s54, s18, 16
+; SI-NEXT: s_lshr_b32 s34, s17, 16
+; SI-NEXT: s_lshr_b32 s53, s16, 16
+; SI-NEXT: s_lshr_b32 s52, s85, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
+; SI-NEXT: s_lshr_b32 s51, s82, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s50, s71, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s49, s68, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB57_4
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s30, 16
+; SI-NEXT: s_lshl_b32 s7, s34, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s58, s51, 16
-; SI-NEXT: s_or_b32 s59, s5, s7
+; SI-NEXT: s_lshl_b32 s62, s53, 16
+; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s31, 16
-; SI-NEXT: s_or_b32 s12, s4, s58
+; SI-NEXT: s_lshl_b32 s7, s35, 16
+; SI-NEXT: s_or_b32 s56, s4, s62
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s52, 16
-; SI-NEXT: s_or_b32 s61, s5, s7
+; SI-NEXT: s_lshl_b32 s72, s54, 16
+; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
-; SI-NEXT: s_or_b32 s10, s4, s60
+; SI-NEXT: s_lshl_b32 s7, s36, 16
+; SI-NEXT: s_or_b32 s46, s4, s72
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s53, 16
-; SI-NEXT: s_or_b32 s63, s5, s7
+; SI-NEXT: s_lshl_b32 s74, s55, 16
+; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s35, 16
-; SI-NEXT: s_or_b32 s8, s4, s62
+; SI-NEXT: s_lshl_b32 s7, s37, 16
+; SI-NEXT: s_or_b32 s44, s4, s74
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s72, s54, 16
-; SI-NEXT: s_or_b32 s73, s5, s7
+; SI-NEXT: s_lshl_b32 s76, s64, 16
+; SI-NEXT: s_or_b32 s77, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s36, 16
-; SI-NEXT: s_or_b32 s6, s4, s72
+; SI-NEXT: s_lshl_b32 s7, s38, 16
+; SI-NEXT: s_or_b32 s42, s4, s76
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s55, 16
-; SI-NEXT: s_or_b32 s57, s5, s7
+; SI-NEXT: s_lshl_b32 s78, s65, 16
+; SI-NEXT: s_or_b32 s79, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s37, 16
-; SI-NEXT: s_or_b32 s4, s4, s56
-; SI-NEXT: s_lshl_b32 s46, s64, 16
-; SI-NEXT: s_or_b32 s47, s5, s7
-; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s38, 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[56:57], 16
-; SI-NEXT: s_and_b32 s56, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s65, 16
-; SI-NEXT: s_or_b32 s45, s5, s7
-; SI-NEXT: s_and_b32 s5, s66, 0xffff
; SI-NEXT: s_lshl_b32 s7, s39, 16
-; SI-NEXT: s_or_b32 s56, s56, s46
-; SI-NEXT: s_lshr_b64 s[76:77], s[46:47], 16
-; SI-NEXT: s_and_b32 s46, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s67, 16
-; SI-NEXT: s_or_b32 s43, s5, s7
-; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_or_b32 s40, s4, s78
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s88, s66, 16
+; SI-NEXT: s_or_b32 s89, s5, s7
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s48, 16
-; SI-NEXT: s_or_b32 s46, s46, s44
-; SI-NEXT: s_lshr_b64 s[78:79], s[44:45], 16
-; SI-NEXT: s_and_b32 s44, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s70, 16
-; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_or_b32 s14, s4, s88
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s90, s67, 16
+; SI-NEXT: s_or_b32 s91, s5, s7
+; SI-NEXT: s_and_b32 s5, s68, 0xffff
; SI-NEXT: s_lshl_b32 s7, s49, 16
-; SI-NEXT: s_or_b32 s44, s44, s42
-; SI-NEXT: s_lshr_b64 s[88:89], s[42:43], 16
-; SI-NEXT: s_and_b32 s42, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s81, 16
-; SI-NEXT: s_or_b32 s15, s5, s7
-; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_or_b32 s12, s4, s90
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s92, s69, 16
+; SI-NEXT: s_or_b32 s93, s5, s7
+; SI-NEXT: s_and_b32 s5, s71, 0xffff
; SI-NEXT: s_lshl_b32 s7, s50, 16
-; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_lshr_b64 s[90:91], s[40:41], 16
-; SI-NEXT: s_and_b32 s40, s82, 0xffff
+; SI-NEXT: s_or_b32 s10, s4, s92
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s94, s80, 16
; SI-NEXT: s_or_b32 s95, s5, s7
-; SI-NEXT: s_lshl_b32 s94, s84, 16
-; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
-; SI-NEXT: s_and_b32 s14, s85, 0xffff
-; SI-NEXT: s_mov_b32 s13, s59
-; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: s_mov_b32 s11, s61
-; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
-; SI-NEXT: s_mov_b32 s9, s63
-; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
-; SI-NEXT: s_mov_b32 s7, s73
-; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
-; SI-NEXT: s_mov_b32 s5, s57
-; SI-NEXT: s_mov_b32 s57, s47
-; SI-NEXT: s_mov_b32 s47, s45
-; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_mov_b32 s43, s41
-; SI-NEXT: s_mov_b32 s41, s15
-; SI-NEXT: s_or_b32 s14, s14, s94
-; SI-NEXT: s_mov_b32 s15, s95
-; SI-NEXT: s_lshr_b64 s[94:95], s[94:95], 16
+; SI-NEXT: s_and_b32 s5, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s51, 16
+; SI-NEXT: s_or_b32 s8, s4, s94
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s30, s83, 16
+; SI-NEXT: s_or_b32 s31, s5, s7
+; SI-NEXT: s_and_b32 s5, s85, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s52, 16
+; SI-NEXT: s_or_b32 s6, s4, s30
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 vcc_lo, s86, 16
+; SI-NEXT: s_or_b32 vcc_hi, s5, s7
+; SI-NEXT: s_or_b32 s4, s4, vcc_lo
+; SI-NEXT: s_lshr_b64 s[58:59], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
+; SI-NEXT: s_mov_b32 s57, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[74:75], 16
+; SI-NEXT: s_mov_b32 s47, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[76:77], 16
+; SI-NEXT: s_mov_b32 s45, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[78:79], 16
+; SI-NEXT: s_mov_b32 s43, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[88:89], 16
+; SI-NEXT: s_mov_b32 s41, s79
+; SI-NEXT: s_lshr_b64 s[78:79], s[90:91], 16
+; SI-NEXT: s_mov_b32 s15, s89
+; SI-NEXT: s_lshr_b64 s[88:89], s[92:93], 16
+; SI-NEXT: s_mov_b32 s13, s91
+; SI-NEXT: s_lshr_b64 s[90:91], s[94:95], 16
+; SI-NEXT: s_mov_b32 s11, s93
+; SI-NEXT: s_lshr_b64 s[92:93], s[30:31], 16
+; SI-NEXT: s_mov_b32 s9, s95
+; SI-NEXT: s_lshr_b64 s[94:95], vcc, 16
+; SI-NEXT: s_mov_b32 s7, s31
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_mov_b32 s5, vcc_hi
; SI-NEXT: s_cbranch_execnz .LBB57_3
; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_add_i32 s87, s87, 3
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_add_i32 s85, s85, 3
-; SI-NEXT: s_and_b32 s4, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
-; SI-NEXT: s_add_i32 s14, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s85, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s52, 16
+; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s83, 16
; SI-NEXT: s_add_i32 s82, s82, 3
-; SI-NEXT: s_add_i32 s15, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
-; SI-NEXT: s_add_i32 s40, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s49, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s51, 16
+; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s80, 16
; SI-NEXT: s_add_i32 s71, s71, 3
-; SI-NEXT: s_add_i32 s41, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
-; SI-NEXT: s_add_i32 s42, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s48, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s50, 16
+; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s10, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s69, 16
; SI-NEXT: s_add_i32 s68, s68, 3
-; SI-NEXT: s_add_i32 s43, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s67, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: s_add_i32 s44, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s39, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s49, 16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_add_i32 s45, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s12, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s67, 16
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s46, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s38, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_and_b32 s13, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s48, 16
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_add_i32 s47, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_and_b32 s14, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s66, 16
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_add_i32 s56, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s37, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_and_b32 s15, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s39, 16
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: s_or_b32 s15, s26, s15
+; SI-NEXT: s_and_b32 s24, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s65, 16
+; SI-NEXT: s_or_b32 s24, s26, s24
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s36, 16
+; SI-NEXT: s_add_i32 s40, s24, 0x30000
+; SI-NEXT: s_and_b32 s24, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s25, s38, 16
+; SI-NEXT: s_or_b32 s24, s25, s24
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s54, 16
+; SI-NEXT: s_add_i32 s41, s24, 0x30000
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s64, 16
+; SI-NEXT: s_or_b32 s22, s24, s22
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s35, 16
+; SI-NEXT: s_add_i32 s42, s22, 0x30000
+; SI-NEXT: s_and_b32 s22, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s37, 16
+; SI-NEXT: s_or_b32 s22, s23, s22
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s53, 16
+; SI-NEXT: s_add_i32 s43, s22, 0x30000
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s55, 16
+; SI-NEXT: s_or_b32 s20, s22, s20
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s34, 16
+; SI-NEXT: s_add_i32 s44, s20, 0x30000
+; SI-NEXT: s_and_b32 s20, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s36, 16
+; SI-NEXT: s_or_b32 s20, s21, s20
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s52, 16
+; SI-NEXT: s_add_i32 s45, s20, 0x30000
+; SI-NEXT: s_and_b32 s18, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s54, 16
+; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s31, 16
+; SI-NEXT: s_add_i32 s46, s18, 0x30000
+; SI-NEXT: s_and_b32 s18, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s35, 16
+; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s51, 16
+; SI-NEXT: s_add_i32 s47, s18, 0x30000
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s53, 16
+; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s30, 16
-; SI-NEXT: s_or_b32 s13, s16, s13
+; SI-NEXT: s_add_i32 s56, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s34, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
@@ -34105,119 +34105,124 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[56:57], 16
-; SI-NEXT: s_lshr_b64 s[78:79], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[92:93], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[94:95], s[14:15], 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s11, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s7, 16
-; SI-NEXT: s_lshr_b32 s36, s5, 16
-; SI-NEXT: s_lshr_b32 s37, s57, 16
-; SI-NEXT: s_lshr_b32 s38, s47, 16
-; SI-NEXT: s_lshr_b32 s39, s45, 16
-; SI-NEXT: s_lshr_b32 s48, s43, 16
-; SI-NEXT: s_lshr_b32 s49, s41, 16
-; SI-NEXT: s_lshr_b32 s50, s15, 16
+; SI-NEXT: s_add_i32 s14, s14, 0x30000
+; SI-NEXT: s_add_i32 s15, s15, 0x30000
+; SI-NEXT: s_add_i32 s57, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[58:59], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s34, s57, 16
+; SI-NEXT: s_lshr_b32 s35, s47, 16
+; SI-NEXT: s_lshr_b32 s36, s45, 16
+; SI-NEXT: s_lshr_b32 s37, s43, 16
+; SI-NEXT: s_lshr_b32 s38, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s15, 16
+; SI-NEXT: s_lshr_b32 s48, s13, 16
+; SI-NEXT: s_lshr_b32 s49, s11, 16
+; SI-NEXT: s_lshr_b32 s50, s9, 16
+; SI-NEXT: s_lshr_b32 s51, s7, 16
+; SI-NEXT: s_lshr_b32 s52, s5, 16
; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s58, 16
-; SI-NEXT: s_or_b32 s12, s12, s16
-; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s30, 16
-; SI-NEXT: s_or_b32 s13, s13, s16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s60, 16
-; SI-NEXT: s_or_b32 s10, s10, s16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s31, 16
-; SI-NEXT: s_or_b32 s11, s11, s16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s62, 16
-; SI-NEXT: s_or_b32 s8, s8, s16
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s34, 16
-; SI-NEXT: s_or_b32 s9, s9, s16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s72, 16
-; SI-NEXT: s_or_b32 s6, s6, s16
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s35, 16
-; SI-NEXT: s_or_b32 s7, s7, s16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s74, 16
-; SI-NEXT: s_or_b32 s4, s4, s16
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s36, 16
-; SI-NEXT: s_or_b32 s5, s5, s16
; SI-NEXT: s_and_b32 s16, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s76, 16
+; SI-NEXT: s_lshl_b32 s17, s58, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s37, 16
+; SI-NEXT: s_lshl_b32 s18, s34, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s78, 16
+; SI-NEXT: s_lshl_b32 s19, s60, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s38, 16
+; SI-NEXT: s_lshl_b32 s20, s35, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s44, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s88, 16
+; SI-NEXT: s_lshl_b32 s21, s62, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s39, 16
+; SI-NEXT: s_lshl_b32 s22, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: s_lshl_b32 s23, s72, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s48, 16
+; SI-NEXT: s_lshl_b32 s24, s37, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s92, 16
+; SI-NEXT: s_lshl_b32 s25, s74, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s49, 16
+; SI-NEXT: s_lshl_b32 s26, s38, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s94, 16
+; SI-NEXT: s_lshl_b32 s26, s76, 16
; SI-NEXT: s_or_b32 s14, s14, s26
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s50, 16
+; SI-NEXT: s_lshl_b32 s26, s39, 16
; SI-NEXT: s_or_b32 s15, s15, s26
-; SI-NEXT: v_readlane_b32 s30, v22, 28
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: v_mov_b32_e32 v4, s8
-; SI-NEXT: v_mov_b32_e32 v5, s9
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s16
-; SI-NEXT: v_mov_b32_e32 v11, s17
-; SI-NEXT: v_mov_b32_e32 v12, s18
-; SI-NEXT: v_mov_b32_e32 v13, s19
-; SI-NEXT: v_mov_b32_e32 v14, s20
-; SI-NEXT: v_mov_b32_e32 v15, s21
-; SI-NEXT: v_mov_b32_e32 v16, s22
-; SI-NEXT: v_mov_b32_e32 v17, s23
-; SI-NEXT: v_mov_b32_e32 v18, s24
-; SI-NEXT: v_mov_b32_e32 v19, s25
-; SI-NEXT: v_mov_b32_e32 v20, s14
-; SI-NEXT: v_mov_b32_e32 v21, s15
-; SI-NEXT: v_readlane_b32 s31, v22, 29
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s78, 16
+; SI-NEXT: s_or_b32 s12, s12, s26
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s48, 16
+; SI-NEXT: s_or_b32 s13, s13, s26
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s88, 16
+; SI-NEXT: s_or_b32 s10, s10, s26
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s49, 16
+; SI-NEXT: s_or_b32 s11, s11, s26
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s90, 16
+; SI-NEXT: s_or_b32 s8, s8, s26
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s50, 16
+; SI-NEXT: s_or_b32 s9, s9, s26
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s92, 16
+; SI-NEXT: s_or_b32 s6, s6, s26
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s51, 16
+; SI-NEXT: s_or_b32 s7, s7, s26
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s94, 16
+; SI-NEXT: s_or_b32 s4, s4, s26
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s52, 16
+; SI-NEXT: s_or_b32 s5, s5, s26
+; SI-NEXT: v_readlane_b32 s30, v22, 30
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s14
+; SI-NEXT: v_mov_b32_e32 v11, s15
+; SI-NEXT: v_mov_b32_e32 v12, s12
+; SI-NEXT: v_mov_b32_e32 v13, s13
+; SI-NEXT: v_mov_b32_e32 v14, s10
+; SI-NEXT: v_mov_b32_e32 v15, s11
+; SI-NEXT: v_mov_b32_e32 v16, s8
+; SI-NEXT: v_mov_b32_e32 v17, s9
+; SI-NEXT: v_mov_b32_e32 v18, s6
+; SI-NEXT: v_mov_b32_e32 v19, s7
+; SI-NEXT: v_mov_b32_e32 v20, s4
+; SI-NEXT: v_mov_b32_e32 v21, s5
+; SI-NEXT: v_readlane_b32 s31, v22, 31
+; SI-NEXT: v_readlane_b32 s87, v22, 29
+; SI-NEXT: v_readlane_b32 s86, v22, 28
; SI-NEXT: v_readlane_b32 s85, v22, 27
; SI-NEXT: v_readlane_b32 s84, v22, 26
; SI-NEXT: v_readlane_b32 s83, v22, 25
@@ -34252,27 +34257,28 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: s_branch .LBB57_2
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index fa1c1854c1d9a..238b3ff3a232a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2754,42 +2754,46 @@ define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {
; SDAG-LABEL: load_v32i8:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: buffer_load_dwordx4 v[36:39], off, s[16:19], 0
-; SDAG-NEXT: buffer_load_dwordx4 v[32:35], off, s[16:19], 0 offset:16
+; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0
+; SDAG-NEXT: buffer_load_dwordx4 v[16:19], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_lshrrev_b64 v[19:20], 24, v[32:33]
-; SDAG-NEXT: v_lshrrev_b64 v[27:28], 24, v[34:35]
-; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v36
-; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v36
-; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v38
-; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v38
-; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v17, 8, v32
-; SDAG-NEXT: v_lshrrev_b32_e32 v18, 16, v32
-; SDAG-NEXT: v_lshrrev_b32_e32 v21, 8, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v22, 16, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v23, 24, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v25, 8, v34
-; SDAG-NEXT: v_lshrrev_b32_e32 v26, 16, v34
-; SDAG-NEXT: v_lshrrev_b32_e32 v29, 8, v35
-; SDAG-NEXT: v_lshrrev_b32_e32 v30, 16, v35
-; SDAG-NEXT: v_lshrrev_b32_e32 v31, 24, v35
-; SDAG-NEXT: v_mov_b32_e32 v0, v36
-; SDAG-NEXT: v_mov_b32_e32 v4, v37
-; SDAG-NEXT: v_mov_b32_e32 v8, v38
-; SDAG-NEXT: v_mov_b32_e32 v12, v39
-; SDAG-NEXT: v_mov_b32_e32 v16, v32
-; SDAG-NEXT: v_mov_b32_e32 v20, v33
-; SDAG-NEXT: v_mov_b32_e32 v24, v34
-; SDAG-NEXT: v_mov_b32_e32 v28, v35
+; SDAG-NEXT: v_lshrrev_b64 v[33:34], 24, v[0:1]
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_lshrrev_b64 v[34:35], 24, v[16:17]
+; SDAG-NEXT: v_lshrrev_b32_e32 v38, 8, v0
+; SDAG-NEXT: v_lshrrev_b32_e32 v32, 16, v0
+; SDAG-NEXT: v_lshrrev_b32_e32 v36, 8, v16
+; SDAG-NEXT: v_lshrrev_b32_e32 v37, 16, v16
+; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[2:3]
+; SDAG-NEXT: v_lshrrev_b64 v[27:28], 24, v[18:19]
+; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; SDAG-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; SDAG-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; SDAG-NEXT: v_mov_b32_e32 v4, v1
+; SDAG-NEXT: v_mov_b32_e32 v8, v2
+; SDAG-NEXT: v_mov_b32_e32 v12, v3
+; SDAG-NEXT: v_mov_b32_e32 v20, v17
+; SDAG-NEXT: v_mov_b32_e32 v24, v18
+; SDAG-NEXT: v_mov_b32_e32 v28, v19
+; SDAG-NEXT: v_mov_b32_e32 v3, v33
+; SDAG-NEXT: v_mov_b32_e32 v19, v34
+; SDAG-NEXT: v_mov_b32_e32 v1, v38
+; SDAG-NEXT: v_mov_b32_e32 v2, v32
+; SDAG-NEXT: v_mov_b32_e32 v17, v36
+; SDAG-NEXT: v_mov_b32_e32 v18, v37
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: load_v32i8:
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 9a6dbcd30e2ea..1042532193944 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -5335,10 +5335,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
@@ -5568,10 +5568,10 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
@@ -6906,10 +6906,10 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 57b653c29ae68..1c7e18d154c36 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -626,12 +626,12 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, 44
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[18:21], v[16:17]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[22:25], v[12:13]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[14:15]
; GFX942-SDAG-NEXT: ; kill: killed $vgpr12_vgpr13
-; GFX942-SDAG-NEXT: ; kill: killed $vgpr14_vgpr15
; GFX942-SDAG-NEXT: ; kill: killed $vgpr16_vgpr17
; GFX942-SDAG-NEXT: s_nop 0
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[8:9]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[14:15]
+; GFX942-SDAG-NEXT: s_nop 0
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[8:9]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[30:33], v[10:11]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[48:51], v[6:7]
@@ -650,10 +650,10 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[6:7]
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[26:29] offset:32
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[48:51]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[52:55]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir b/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
index 9e38919190ea7..086fb4374f9bc 100644
--- a/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
+++ b/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
@@ -14,12 +14,10 @@ body: |
; REG_ALLOC-NEXT: renamable $vgpr11_vgpr12_vgpr13_vgpr14 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr3, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr2, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $vgpr5_vgpr6_vgpr7_vgpr8 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr0, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; REG_ALLOC-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr4, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; REG_ALLOC-NEXT: KILL killed renamable $vgpr4
; REG_ALLOC-NEXT: KILL killed renamable $vgpr2
; REG_ALLOC-NEXT: KILL killed renamable $vgpr0
; REG_ALLOC-NEXT: KILL killed renamable $vgpr3
- ; REG_ALLOC-NEXT: KILL killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
+ ; REG_ALLOC-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr4, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $sgpr8 = V_READFIRSTLANE_B32 killed $vgpr11, implicit $exec
; REG_ALLOC-NEXT: renamable $sgpr9 = V_READFIRSTLANE_B32 killed $vgpr12, implicit $exec
; REG_ALLOC-NEXT: renamable $sgpr6_sgpr7 = V_CMP_NE_U32_e64 killed $vgpr1, 0, implicit $exec
@@ -83,12 +81,10 @@ body: |
; DEAD_INST_DEL-NEXT: renamable $vgpr11_vgpr12_vgpr13_vgpr14 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr3, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr2, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $vgpr5_vgpr6_vgpr7_vgpr8 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr0, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; DEAD_INST_DEL-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr4, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr4
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr2
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr0
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr3
- ; DEAD_INST_DEL-NEXT: KILL killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
+ ; DEAD_INST_DEL-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr4, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $sgpr8 = V_READFIRSTLANE_B32 killed $vgpr11, implicit $exec
; DEAD_INST_DEL-NEXT: renamable $sgpr9 = V_READFIRSTLANE_B32 killed $vgpr12, implicit $exec
; DEAD_INST_DEL-NEXT: renamable $sgpr6_sgpr7 = V_CMP_NE_U32_e64 killed $vgpr1, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.mir b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
index e50c3146068f3..d8647d1009053 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
@@ -424,9 +424,10 @@ body: |
# GCN-LABEL: {{^}}name: ptr_use_after_clause_subreg_multi{{$}}
# GCN: dead %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub0_sub1, 0, 0, implicit $exec
# GCN-NEXT: dead %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub2_sub3, 16, 0, implicit $exec
+# GCN-NEXT: KILL %0.sub2{{$}}
# GCN-NEXT: dead %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub3_sub4, 32, 0, implicit $exec
# GCN-NEXT: dead %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub6_sub7, 48, 0, implicit $exec
-# GCN-NEXT: KILL %0.sub2_sub3_sub4, %0.sub7{{$}}
+# GCN-NEXT: KILL %0.sub3_sub4, %0.sub7{{$}}
# GCN-NEXT: S_NOP 0, implicit %0.sub0_sub1, implicit %0.sub5_sub6
---
name: ptr_use_after_clause_subreg_multi
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
index 5255eb3bed12d..e2b4e19bada87 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
@@ -197,6 +197,8 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: ; def s21
; GFX906-NEXT: ;;#ASMEND
; GFX906-NEXT: v_writelane_b32 v39, s21, 12
+; GFX906-NEXT: s_or_saveexec_b64 s[34:35], -1
+; GFX906-NEXT: s_mov_b64 exec, s[34:35]
; GFX906-NEXT: ;;#ASMSTART
; GFX906-NEXT: ; def s22
; GFX906-NEXT: ;;#ASMEND
@@ -237,10 +239,10 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: v_readlane_b32 s8, v39, 6
; GFX906-NEXT: v_readlane_b32 s10, v39, 4
; GFX906-NEXT: v_readlane_b32 s16, v39, 22
+; GFX906-NEXT: v_mov_b32_e32 v31, v40
; GFX906-NEXT: v_readlane_b32 s12, v39, 3
; GFX906-NEXT: v_readlane_b32 s13, v39, 2
; GFX906-NEXT: v_readlane_b32 s14, v39, 1
-; GFX906-NEXT: v_mov_b32_e32 v31, v40
; GFX906-NEXT: v_readlane_b32 s15, v39, 0
; GFX906-NEXT: v_readlane_b32 s5, v39, 11
; GFX906-NEXT: v_readlane_b32 s7, v39, 9
@@ -579,6 +581,8 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: ; def s21
; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: v_writelane_b32 v39, s21, 12
+; GFX908-NEXT: s_or_saveexec_b64 s[34:35], -1
+; GFX908-NEXT: s_mov_b64 exec, s[34:35]
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def s22
; GFX908-NEXT: ;;#ASMEND
@@ -619,10 +623,10 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: v_readlane_b32 s8, v39, 6
; GFX908-NEXT: v_readlane_b32 s10, v39, 4
; GFX908-NEXT: v_readlane_b32 s16, v39, 22
+; GFX908-NEXT: v_mov_b32_e32 v31, v40
; GFX908-NEXT: v_readlane_b32 s12, v39, 3
; GFX908-NEXT: v_readlane_b32 s13, v39, 2
; GFX908-NEXT: v_readlane_b32 s14, v39, 1
-; GFX908-NEXT: v_mov_b32_e32 v31, v40
; GFX908-NEXT: v_readlane_b32 s15, v39, 0
; GFX908-NEXT: v_readlane_b32 s5, v39, 11
; GFX908-NEXT: v_readlane_b32 s7, v39, 9
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
index 516db40a5af76..f4c098ec0464e 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
# Test that the upward register pressure tracker accounts for early-clobber
# physical register defs overlapping with physical register uses.
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
index bc6f1e1888bbc..476ab04e2a120 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
@@ -1,7 +1,7 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s 2>&1 | FileCheck %s --check-prefix=RPU
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward %s 2>&1 | FileCheck %s --check-prefix=RPD
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-track-physregs-in-gcn-trackers=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
# Tests for physical register pressure tracking edge cases:
# 1. Non-allocatable registers (e.g. $scc) should not affect pressure.
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
index b41aa088bfdde..f424035f14824 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
@@ -161,7 +161,7 @@ body: |
; RPU: bb.0:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 0 1
; RPU-NEXT: 0 1 %0:vgpr_32 = COPY $vgpr0
; RPU-NEXT: 0 1
; RPU-NEXT: 0 3 %1:vreg_64 = IMPLICIT_DEF
@@ -272,8 +272,8 @@ body: |
; RPD: bb.0:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 %0:vgpr_32 = COPY $vgpr0
+ ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 0 2 %0:vgpr_32 = COPY $vgpr0
; RPD-NEXT: 0 1
; RPD-NEXT: 0 3 %1:vreg_64 = IMPLICIT_DEF
; RPD-NEXT: 0 3
@@ -498,35 +498,35 @@ body: |
; RPU: bb.0:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0 = COPY $sgpr1
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr2_sgpr3 = S_GETPC_B64
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr1 = COPY killed $sgpr3
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_BRANCH %bb.1
- ; RPU-NEXT: 0 1
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 0 $sgpr0 = COPY $sgpr1
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 2 0 $sgpr2_sgpr3 = S_GETPC_B64
+ ; RPU-NEXT: 2 0
+ ; RPU-NEXT: 2 0 $sgpr1 = COPY killed $sgpr3
+ ; RPU-NEXT: 2 0
+ ; RPU-NEXT: 4 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 4 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_BRANCH %bb.1
+ ; RPU-NEXT: 1 1
; RPU-NEXT: Live-out: %0:0000000000000C00
; RPU-NEXT: Live-thr:
; RPU-NEXT: 0 0
; RPU-NEXT: bb.1:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 $m0 = S_MOV_B32 killed $sgpr0
; RPU-NEXT: 0 1
; RPU-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
; RPU-NEXT: 0 1
@@ -551,38 +551,38 @@ body: |
; RPD: bb.0:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0 = COPY $sgpr1
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr2_sgpr3 = S_GETPC_B64
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr1 = COPY killed $sgpr3
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_BRANCH %bb.1
- ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 2 0 $sgpr0 = COPY $sgpr1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 3 0 $sgpr2_sgpr3 = S_GETPC_B64
+ ; RPD-NEXT: 3 0
+ ; RPD-NEXT: 4 0 $sgpr1 = COPY killed $sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 4 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 4 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
+ ; RPD-NEXT: 3 0
+ ; RPD-NEXT: 3 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_BRANCH %bb.1
+ ; RPD-NEXT: 3 1
; RPD-NEXT: Live-out: %0:0000000000000C00
; RPD-NEXT: Live-thr:
; RPD-NEXT: 0 0
; RPD-NEXT: bb.1:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
- ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPD-NEXT: 1 1
+ ; RPD-NEXT: 1 1 $m0 = S_MOV_B32 killed $sgpr0
+ ; RPD-NEXT: 1 1
+ ; RPD-NEXT: 1 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
+ ; RPD-NEXT: 1 1
; RPD-NEXT: Live-out: %0:0000000000000C00
; RPD-NEXT: Live-thr:
; RPD-NEXT: 0 0
@@ -632,7 +632,7 @@ body: |
; RPU-LABEL: name: test_partially_used_def
; RPU: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %1:sgpr_128 = COPY %0:sgpr_128
@@ -646,8 +646,8 @@ body: |
; RPD-LABEL: name: test_partially_used_def
; RPD: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPD-NEXT: 4 0
; RPD-NEXT: 8 0 %1:sgpr_128 = COPY %0:sgpr_128
; RPD-NEXT: 1 0
@@ -666,19 +666,33 @@ tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr0_sgpr1_sgpr2_sgpr3
- ; RP-LABEL: name: test_partially_used_early_clobber_def
- ; RP: Live-in:
- ; RP-NEXT: SGPR VGPR
- ; RP-NEXT: 0 0
- ; RP-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
- ; RP-NEXT: 4 0
- ; RP-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
- ; RP-NEXT: 1 0
- ; RP-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
- ; RP-NEXT: 0 0
- ; RP-NEXT: Live-out:
- ; RP-NEXT: Live-thr:
- ; RP-NEXT: 0 0
+ ; RPU-LABEL: name: test_partially_used_early_clobber_def
+ ; RPU: Live-in:
+ ; RPU-NEXT: SGPR VGPR
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
+ ; RPU-NEXT: 0 0
+ ; RPU-NEXT: Live-out:
+ ; RPU-NEXT: Live-thr:
+ ; RPU-NEXT: 0 0
+ ;
+ ; RPD-LABEL: name: test_partially_used_early_clobber_def
+ ; RPD: Live-in:
+ ; RPD-NEXT: SGPR VGPR
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
+ ; RPD-NEXT: 0 0
+ ; RPD-NEXT: Live-out:
+ ; RPD-NEXT: Live-thr:
+ ; RPD-NEXT: 0 0
%0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
early-clobber %1:sgpr_128 = COPY %0:sgpr_128
S_NOP 0, implicit %1.sub1
@@ -692,7 +706,7 @@ body: |
; RPU-LABEL: name: test_partially_used_def_and_early_clobber_def
; RPU: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPU-NEXT: 4 0
; RPU-NEXT: 16 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128
@@ -706,8 +720,8 @@ body: |
; RPD-LABEL: name: test_partially_used_def_and_early_clobber_def
; RPD: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPD-NEXT: 4 0
; RPD-NEXT: 20 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128
; RPD-NEXT: 6 0
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
index d6e3b6a734eec..906547512f8f4 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
@@ -3,7 +3,7 @@
# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
-# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 \
# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
index 0be216846f4be..1d84311e385f7 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
@@ -3,7 +3,7 @@
# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
-# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 \
# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index b2b73e9a96fcb..f54ed99c5035d 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,6 +1,6 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index 91a071dd69049..47de1b20f6703 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -15,18 +15,19 @@
;
; Check scheduling pressure values:
; SCHED-LABEL: spill:%bb.0 entry
-; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 98
-; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 97
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
+; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
;
; SCHED-GCNTRACKERS-LABEL: spill:%bb.0 entry
; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
; SCHED-GCNTRACKERS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
;
-; NOTE: GCN Trackers now track pressure from both virtual and physical registers.
-; The GCN tracker now matches the generic tracker's VGPR count (1 VGPR).
-; When a live range is not found for a physical regunit, we conservatively
-; assume the unit is live, so Region SGPR pressure can be higher (193 vs 98).
-; Pressure after scheduling remains 98 vs 97 due to physical register tracking.
+; NOTE: Physical register pressure tracking is now decoupled from the GCN
+; trackers, so both the generic and GCN-tracker scheduling paths account for
+; physical registers. As a result SCHED and SCHED-GCNTRACKERS now report the
+; same pressure. When a live range is not found for a physical regunit, we
+; conservatively assume the unit is live, so Region SGPR pressure is higher
+; (193) than the virtual-only count (98).
define amdgpu_kernel void @spill(ptr addrspace(1) %arg, i32 %cnd) #0 {
entry:
@@ -267,7 +268,7 @@ bb3:
; GCN-GCNTRACKERS: ScratchSize: 8
;
; SCHED-LABEL: spill_func:%bb.0 entry
-; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 97
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
;
; SCHED-GCNTRACKERS-LABEL: spill_func:%bb.0 entry
; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
index 7c947f4a942a6..cc7ad68fc6546 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
@@ -3,7 +3,7 @@
; RUN: FileCheck --check-prefix=GCN-DEBUG %s < %t
; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=NO-GCN %s
; RUN: FileCheck --check-prefix=GENERIC-DEBUG %s < %t
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
; RUN: FileCheck --check-prefix=GCN-NOPHYS-DEBUG %s < %t
; REQUIRES: asserts
@@ -14,8 +14,8 @@
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_single_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_single_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -73,7 +73,7 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_multiple_physregs
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_multiple_physregs
@@ -138,8 +138,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
; GENERIC-DEBUG-LABEL: test_physreg_with_vreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 10, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
; GCN-NOPHYS-DEBUG-LABEL: test_physreg_with_vreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
@@ -217,8 +217,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_early_clobber
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -277,8 +277,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GENERIC-DEBUG-LABEL: test_early_clobber_tuple
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 9
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber_tuple
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
@@ -347,8 +347,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_physreg_input
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_physreg_input
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -407,8 +407,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GENERIC-DEBUG-LABEL: test_tuple_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GCN-NOPHYS-DEBUG-LABEL: test_tuple_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
@@ -468,8 +468,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
; GENERIC-DEBUG-LABEL: test_tuple128_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
; GCN-NOPHYS-DEBUG-LABEL: test_tuple128_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
@@ -528,8 +528,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
; GENERIC-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
-; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 16, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
; GCN-NOPHYS-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
diff --git a/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
index af9ff4bae8292..4cf9575b9d3af 100644
--- a/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
+++ b/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
@@ -28,7 +28,6 @@ body: |
; CHECK-NEXT: DBG_VALUE [[S_LOAD_DWORD_IMM2]], 0, 0
; CHECK-NEXT: DBG_VALUE [[S_LOAD_DWORD_IMM3]], 0, 0
; CHECK-NEXT: [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 64, 0 :: (load (s32), addrspace 4)
- ; CHECK-NEXT: KILL [[COPY]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[S_LOAD_DWORD_IMM]], implicit [[S_LOAD_DWORD_IMM1]], implicit [[S_LOAD_DWORD_IMM2]], implicit [[S_LOAD_DWORD_IMM3]], implicit [[S_LOAD_DWORD_IMM4]]
%0:sreg_64 = COPY $sgpr4_sgpr5
%1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (load (s32), align 4, addrspace 4)
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index c0c1763d54cc0..92ef86800256d 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -46,8 +46,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_]], 16, 0 :: (invariant load (s128) from %ir.83, addrspace 4)
; CHECK-NEXT: early-clobber %73:sgpr_256 = S_LOAD_DWORDX8_IMM_ec undef %74:sreg_64, 0, 0 :: (invariant load (s256) from `ptr addrspace(4) poison`, align 16, addrspace 4)
- ; CHECK-NEXT: KILL [[S_ADD_U32_]].sub0, [[S_ADD_U32_]].sub1
- ; CHECK-NEXT: KILL undef %74:sreg_64
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_LOAD_DWORDX4_IMM]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = S_MOV_B32 0
@@ -55,7 +53,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %94:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], %73.sub0_sub1_sub2_sub3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: KILL undef %94:sgpr_128
- ; CHECK-NEXT: KILL undef %123:sgpr_128
; CHECK-NEXT: [[S_SUB_I32_2:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM1]], 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_1:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
@@ -64,8 +61,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_ADD_U32_3:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.89, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.95, addrspace 4)
- ; CHECK-NEXT: KILL [[S_ADD_U32_2]].sub0, [[S_ADD_U32_2]].sub1
- ; CHECK-NEXT: KILL [[S_ADD_U32_1]].sub0, [[S_ADD_U32_1]].sub1
; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %174:sreg_32, 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %174:sreg_32, implicit-def $scc
@@ -206,8 +201,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_21]], 168, 0 :: (invariant load (s32) from %ir.308, align 8, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM22]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM23]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM22]]
; CHECK-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 %73.sub0, 65535, implicit-def dead $scc
; CHECK-NEXT: [[COPY18:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
; CHECK-NEXT: [[COPY18:%[0-9]+]].sub1:sgpr_128 = COPY [[S_AND_B32_2]]
@@ -231,10 +224,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM24]]
- ; CHECK-NEXT: KILL [[V_MOV_B32_e32_]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM25]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM26]]
; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -2, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -1, [[BUFFER_LOAD_FORMAT_X_IDXEN1]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -3, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
More information about the llvm-branch-commits
mailing list