[llvm-branch-commits] [llvm] [2/3][AMDGPU] Physical register tracking in GCN trackers. (PR #184275)
Dhruva Chakrabarti via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 19 19:01:13 PDT 2026
https://github.com/dhruvachak updated https://github.com/llvm/llvm-project/pull/184275
>From 5a649375383035fa67f30d0fa91b5c8d668a8637 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Thu, 2 Jul 2026 14:59:21 -0500
Subject: [PATCH 1/5] [AMDGPU] Add physical register support in GCN trackers
Squashed physical-register tracking work (28 commits) onto the virtual
rename base, to be rebased as a single unit.
Includes:
- Physical register live-in/live-out seeding in GCN RP trackers
- LiveRegUnits-based physical register tracking
- Early-clobber handling for physical registers
- Unified GCNRegPressure objects for virtual and physical
- Tests for physical liveins/liveouts, early clobber, aliasing/allocatable
Assisted-by: Cursor/Claude Opus
---
.../Target/AMDGPU/AMDGPUNextUseAnalysis.cpp | 2 +-
.../Target/AMDGPU/GCNIterativeScheduler.cpp | 18 +-
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 405 +++--
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 147 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 50 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 6 +
.../lib/Target/AMDGPU/SIFormMemoryClauses.cpp | 11 +-
.../machine-scheduler-sink-trivial-remats.mir | 20 +-
.../AMDGPU/materialize-frame-index-sgpr.ll | 1575 ++++++++++++++++-
.../regpressure-physreg-early-clobber.mir | 43 +
.../AMDGPU/regpressure-physreg-limits.mir | 239 +++
.../CodeGen/AMDGPU/sched-physreg-liveins.mir | 68 +
.../CodeGen/AMDGPU/sched-physreg-liveouts.mir | 84 +
.../schedule-amdgpu-tracker-physreg-crash.ll | 16 +-
.../AMDGPU/schedule-amdgpu-tracker-physreg.ll | 37 +-
.../AMDGPU/schedule-gcn-physreg-pressure.ll | 630 +++++++
.../Target/AMDGPU/GCNRegPressureTest.cpp | 9 +-
17 files changed, 3176 insertions(+), 184 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
index 135afc26eef1f..7600d422cc2f1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUNextUseAnalysis.cpp
@@ -2431,7 +2431,7 @@ void printNextUseDistancesAsJson(json::OStream &J, const MachineFunction &MF,
// We don't actually care about register pressure here - just using
// GCNDownwardRPTracker as a convenient way of getting the set of live
// registers at a given instruction.
- GCNDownwardRPTracker RPTracker(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MRI);
ModuleSlotTracker MST(M);
MST.incorporateFunction(F);
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index c4a6ea91fedb6..ca54f8b52a6d4 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -238,11 +238,8 @@ class SchedStrategyStub : public MachineSchedStrategy {
GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
StrategyKind S)
- : BaseClass(C, std::make_unique<SchedStrategyStub>())
- , Context(C)
- , Strategy(S)
- , UPTracker(*LIS) {
-}
+ : BaseClass(C, std::make_unique<SchedStrategyStub>()), Context(C),
+ Strategy(S), UPTracker(*LIS, Context->MF->getRegInfo()) {}
// returns max pressure for a region
GCNRegPressure
@@ -260,7 +257,7 @@ GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
auto AfterBottomMI = std::next(BottomMI);
if (AfterBottomMI == BBEnd ||
&*AfterBottomMI != UPTracker.getLastTrackedMI()) {
- UPTracker.reset(*BottomMI);
+ UPTracker.reset(*BottomMI, Begin->getParent());
} else {
assert(UPTracker.isValid());
}
@@ -280,16 +277,17 @@ GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
template <typename Range> GCNRegPressure
GCNIterativeScheduler::getSchedulePressure(const Region &R,
Range &&Schedule) const {
- auto const BBEnd = R.Begin->getParent()->end();
- GCNUpwardRPTracker RPTracker(*LIS);
+ const MachineBasicBlock *MBB = R.Begin->getParent();
+ auto const BBEnd = MBB->end();
+ GCNUpwardRPTracker RPTracker(*LIS, MF.getRegInfo());
if (R.End != BBEnd) {
// R.End points to the boundary instruction but the
// schedule doesn't include it
- RPTracker.reset(*R.End);
+ RPTracker.reset(*R.End, MBB);
RPTracker.recede(*R.End);
} else {
// R.End doesn't point to the boundary instruction
- RPTracker.reset(*std::prev(BBEnd));
+ RPTracker.reset(*std::prev(BBEnd), MBB);
}
for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
RPTracker.recede(*getMachineInstr(*--I));
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index c378458d8829d..0bfbb6a94b2d8 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -47,10 +47,8 @@ unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,
: (STI->isVectorSuperClass(RC) ? AVGPR : VGPR));
}
-void GCNRegPressure::inc(unsigned Reg,
- LaneBitmask PrevMask,
- LaneBitmask NewMask,
- const MachineRegisterInfo &MRI) {
+void GCNRegPressure::inc(unsigned Reg, LaneBitmask PrevMask,
+ LaneBitmask NewMask, const MachineRegisterInfo &MRI) {
unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(NewMask);
unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(PrevMask);
if (NewNumCoveredRegs == PrevNumCoveredRegs)
@@ -98,6 +96,34 @@ void GCNRegPressure::inc(unsigned Reg,
Value[RegKind] += Sign;
}
+unsigned GCNRegPressure::pressureSetToRegKind(unsigned PSetID) {
+ switch (PSetID) {
+ case AMDGPU::RegisterPressureSets::SReg_32:
+ return SGPR;
+ case AMDGPU::RegisterPressureSets::AGPR_32:
+ return AGPR;
+ case AMDGPU::RegisterPressureSets::VGPR_32:
+ return VGPR;
+ }
+ llvm_unreachable("unexpected pressure set");
+}
+
+// Adjusts both raw count and tuple weight per unit. Raw count and
+// tuple weight receive identical increments. This means 32-bit physical
+// registers contribute to tuple weight (unlike virtual registers where only
+// tuples > 32-bit contribute).
+void GCNRegPressure::adjustPhysUnitPressure(MCRegUnit Unit, bool IsAdd,
+ const SIRegisterInfo &SRI) {
+ const int *PSetIDs = SRI.getRegUnitPressureSets(Unit);
+ if (PSetIDs[0] == -1)
+ return;
+ assert(PSetIDs[1] == -1 && "expected single pressure set per unit");
+ unsigned Kind = pressureSetToRegKind(PSetIDs[0]);
+ int Delta = (IsAdd ? 1 : -1) * static_cast<int>(SRI.getRegUnitWeight(Unit));
+ Value[Kind] += Delta;
+ Value[TOTAL_KINDS + Kind] += Delta;
+}
+
namespace {
struct RegExcess {
unsigned SGPR = 0;
@@ -485,6 +511,60 @@ LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI, LaneMaskFilter);
}
+bool GCNRPTracker::isUnitLiveAt(MCRegUnit Unit, SlotIndex SI) const {
+ const LiveRange *LR = LIS.getCachedRegUnit(Unit);
+ // If LIS has no reg-unit live range, be conservative and assume it is live.
+ return !LR || LR->liveAt(SI);
+}
+
+void GCNRPTracker::addUnitsAndIncPressure(MCRegister Reg,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (!PhysLiveRegUnits.test(U)) {
+ PhysLiveRegUnits.set(U);
+ Pressure.inc(Unit, *SRI);
+ }
+ }
+}
+
+void GCNRPTracker::removeUnitsAndDecPressure(MCRegister Reg,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U)) {
+ PhysLiveRegUnits.reset(U);
+ Pressure.dec(Unit, *SRI);
+ }
+ }
+}
+
+void GCNRPTracker::removeKilledUnitsAndDecPressure(MCRegister Reg, SlotIndex SI,
+ GCNRegPressure &Pressure) {
+ assert(SRI && "SRI not initialized");
+ for (MCRegUnit Unit : SRI->regunits(Reg)) {
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SI)) {
+ PhysLiveRegUnits.reset(U);
+ Pressure.dec(Unit, *SRI);
+ }
+ }
+}
+
+void llvm::addPhysRegPressure(GCNRegPressure &RP, const SIRegisterInfo &SRI,
+ const BitVector &PhysLiveUnits) {
+ for (unsigned U : PhysLiveUnits.set_bits())
+ RP.inc(static_cast<MCRegUnit>(U), SRI);
+}
+
+GCNRegPressure GCNRPTracker::constructPhysRegPressure() const {
+ GCNRegPressure Res;
+ addPhysRegPressure(Res, *SRI, PhysLiveRegUnits);
+ return Res;
+}
+
LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
const MachineRegisterInfo &MRI,
LaneBitmask LaneMaskFilter) {
@@ -557,18 +637,46 @@ void GCNRPTracker::reset(const MachineBasicBlock &MBB, bool End) {
void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
this->MRI = &MRI;
+ SRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
LastTrackedMI = nullptr;
VirtLiveRegs = llvm::getVirtLiveRegs(SI, LIS, MRI);
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
+
+ updatePhysRegTracking();
+ // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
+ // stale data if physical tracking was previously enabled.
+ PhysLiveRegUnits.reset();
}
void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
const LiveRegSet &VirtLiveRegs) {
this->MRI = &MRI;
+ SRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
LastTrackedMI = nullptr;
if (&this->VirtLiveRegs != &VirtLiveRegs)
this->VirtLiveRegs = VirtLiveRegs;
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
+
+ updatePhysRegTracking();
+ // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
+ // stale data if physical tracking was previously enabled.
+ PhysLiveRegUnits.reset();
+}
+
+void GCNRPTracker::reset(const MachineRegisterInfo &MRInfo,
+ const LiveRegSet &VirtLiveRegsSet,
+ const BitVector &PhysLiveUnits) {
+ reset(MRInfo, VirtLiveRegsSet);
+ initPhysLiveUnits(PhysLiveUnits);
+}
+
+void GCNRPTracker::initPhysLiveUnits(const BitVector &PhysLiveUnits) {
+ if (!TrackPhysRegs)
+ return;
+ PhysLiveRegUnits = PhysLiveUnits;
+ GCNRegPressure PhysPressure = constructPhysRegPressure();
+ CurPressure += PhysPressure;
+ MaxPressure = max(MaxPressure, CurPressure);
}
/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
@@ -596,29 +704,37 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
GCNRegPressure DefPressure, ECDefPressure;
bool HasECDefs = false;
for (const MachineOperand &MO : MI.all_defs()) {
- if (!MO.getReg().isVirtual())
- continue;
-
Register Reg = MO.getReg();
- LaneBitmask DefMask = getDefRegMask(MO, *MRI);
- // Treat a def as fully live at the moment of definition: keep a record.
- if (MO.isEarlyClobber()) {
- ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
- HasECDefs = true;
- } else
- DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+ if (Reg.isVirtual()) {
+ LaneBitmask DefMask = getDefRegMask(MO, *MRI);
- auto I = VirtLiveRegs.find(Reg);
- if (I == VirtLiveRegs.end())
- continue;
+ // Treat a def as fully live at the moment of definition: keep a record.
+ if (MO.isEarlyClobber()) {
+ ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+ HasECDefs = true;
+ } else
+ DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
+
+ auto I = VirtLiveRegs.find(Reg);
+ if (I == VirtLiveRegs.end())
+ continue;
+
+ LaneBitmask &LiveMask = I->second;
+ LaneBitmask PrevMask = LiveMask;
+ LiveMask &= ~DefMask;
+ CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ if (LiveMask.none())
+ VirtLiveRegs.erase(I);
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (MO.isEarlyClobber()) {
+ for (MCRegUnit Unit : SRI->regunits(Reg.asMCReg()))
+ ECDefPressure.inc(Unit, *SRI);
+ HasECDefs = true;
+ }
- LaneBitmask &LiveMask = I->second;
- LaneBitmask PrevMask = LiveMask;
- LiveMask &= ~DefMask;
- CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
- if (LiveMask.none())
- VirtLiveRegs.erase(I);
+ removeUnitsAndDecPressure(Reg.asMCReg(), CurPressure);
+ }
}
// Update MaxPressure with defs pressure.
@@ -637,11 +753,31 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
CurPressure.inc(U.VRegOrUnit.asVirtualReg(), PrevMask, LiveMask, *MRI);
}
+ if (TrackPhysRegs) {
+ for (const MachineOperand &MO : MI.all_uses()) {
+ if (!MO.readsReg())
+ continue;
+ Register Reg = MO.getReg();
+ if (!shouldTrackPhysReg(Reg))
+ continue;
+ addUnitsAndIncPressure(Reg.asMCReg(), CurPressure);
+ }
+ }
+
// Update MaxPressure with uses plus early-clobber defs pressure.
MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)
: max(CurPressure, MaxPressure);
- assert(CurPressure == getVirtRegPressure(*MRI, VirtLiveRegs));
+#ifndef NDEBUG
+ auto VirtPressure = getVirtRegPressure(*MRI, VirtLiveRegs);
+ auto PhysPressure = constructPhysRegPressure();
+ assert(CurPressure == VirtPressure + PhysPressure ||
+ (dbgs() << "Pressure mismatch in recede()\nMI: " << MI
+ << "Tracked: " << print(CurPressure) << "Expected: "
+ << print(VirtPressure + PhysPressure) << "Virt: "
+ << print(VirtPressure) << "Phys: " << print(PhysPressure),
+ false));
+#endif
}
////////////////////////////////////////////////////////////////////////////////
@@ -649,7 +785,8 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy) {
+ const LiveRegSet *VirtLiveRegsCopy,
+ const MachineBasicBlock *SeedPhysMBB) {
MBBEnd = MI.getParent()->end();
assert(End == MBBEnd ||
End->getParent()->end() == MBBEnd && "end unrelated to MI block");
@@ -666,9 +803,22 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
GCNRPTracker::reset(*NextMI, /*After=*/false);
else
GCNRPTracker::reset(*MI.getParent(), /*End=*/true);
+
+ if (SeedPhysMBB && TrackPhysRegs &&
+ MI.getMF()->getProperties().hasTracksLiveness())
+ initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveins());
+
return NextMI != End;
}
+bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
+ const LiveRegSet &VirtLiveRegs,
+ const BitVector &PhysLiveUnits) {
+ bool Result = reset(MI, MI.getParent()->end(), &VirtLiveRegs);
+ initPhysLiveUnits(PhysLiveUnits);
+ return Result;
+}
+
bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
bool UseInternalIterator) {
assert(MRI && "call reset first");
@@ -693,38 +843,46 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
// Remove dead registers or mask bits.
SmallSet<Register, 8> SeenRegs;
- for (auto &MO : CurrMI->operands()) {
- if (!MO.isReg() || !MO.getReg().isVirtual())
- continue;
- if (MO.isUse() && !MO.readsReg())
- continue;
- if (!UseInternalIterator && MO.isDef())
- continue;
- if (!SeenRegs.insert(MO.getReg()).second)
+ for (const auto &MO : CurrMI->operands()) {
+ if (!MO.isReg())
continue;
- const LiveInterval &LI = LIS.getInterval(MO.getReg());
- if (LI.hasSubRanges()) {
- auto It = VirtLiveRegs.end();
- for (const auto &S : LI.subranges()) {
- if (!S.liveAt(SI)) {
- if (It == VirtLiveRegs.end()) {
- It = VirtLiveRegs.find(MO.getReg());
- if (It == VirtLiveRegs.end())
- llvm_unreachable("register isn't live");
+ Register Reg = MO.getReg();
+
+ if (Reg.isVirtual()) {
+ if (MO.isUse() && !MO.readsReg())
+ continue;
+ if (!UseInternalIterator && MO.isDef())
+ continue;
+ if (!SeenRegs.insert(Reg).second)
+ continue;
+ const LiveInterval &LI = LIS.getInterval(Reg);
+ if (LI.hasSubRanges()) {
+ auto It = VirtLiveRegs.end();
+ for (const auto &S : LI.subranges()) {
+ if (!S.liveAt(SI)) {
+ if (It == VirtLiveRegs.end()) {
+ It = VirtLiveRegs.find(Reg);
+ if (It == VirtLiveRegs.end())
+ llvm_unreachable("register isn't live");
+ }
+ auto PrevMask = It->second;
+ It->second &= ~S.LaneMask;
+ CurPressure.inc(Reg, PrevMask, It->second, *MRI);
}
- auto PrevMask = It->second;
- It->second &= ~S.LaneMask;
- CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);
}
- }
- if (It != VirtLiveRegs.end() && It->second.none())
+ if (It != VirtLiveRegs.end() && It->second.none())
+ VirtLiveRegs.erase(It);
+ } else if (!LI.liveAt(SI)) {
+ auto It = VirtLiveRegs.find(Reg);
+ if (It == VirtLiveRegs.end())
+ llvm_unreachable("register isn't live");
+ CurPressure.inc(Reg, It->second, LaneBitmask::getNone(), *MRI);
VirtLiveRegs.erase(It);
- } else if (!LI.liveAt(SI)) {
- auto It = VirtLiveRegs.find(MO.getReg());
- if (It == VirtLiveRegs.end())
- llvm_unreachable("register isn't live");
- CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);
- VirtLiveRegs.erase(It);
+ }
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (!SeenRegs.insert(Reg).second)
+ continue;
+ removeKilledUnitsAndDecPressure(Reg.asMCReg(), SI, CurPressure);
}
}
@@ -749,12 +907,16 @@ void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,
// Add new registers or mask bits.
for (const auto &MO : CurrMI->all_defs()) {
Register Reg = MO.getReg();
- if (!Reg.isVirtual())
- continue;
- auto &LiveMask = VirtLiveRegs[Reg];
- auto PrevMask = LiveMask;
- LiveMask |= getDefRegMask(MO, *MRI);
- CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+
+ if (Reg.isVirtual()) {
+ auto &LiveMask = VirtLiveRegs[Reg];
+ auto PrevMask = LiveMask;
+ LiveMask |= getDefRegMask(MO, *MRI);
+ CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ } else if (shouldTrackPhysReg(Reg)) {
+ if (!MO.isDead())
+ addUnitsAndIncPressure(Reg.asMCReg(), CurPressure);
+ }
}
MaxPressure = max(MaxPressure, CurPressure);
@@ -788,10 +950,13 @@ bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy) {
- if (!reset(*Begin, End, VirtLiveRegsCopy))
- return false;
- return advance(End);
+ const LiveRegSet *VirtLiveRegsCopy,
+ const BitVector *PhysLiveUnits) {
+ bool ResetStatus = reset(*Begin, End, VirtLiveRegsCopy);
+ // Restore the physical snapshot regardless of the above status.
+ if (PhysLiveUnits)
+ initPhysLiveUnits(*PhysLiveUnits);
+ return ResetStatus && advance(End);
}
Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
@@ -847,47 +1012,60 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
// Tracks the live mask reported by the use loop for redefined registers.
SmallDenseMap<Register, LaneBitmask, 8> PostUseMask;
+ // Process uses: decrement pressure for last-use lanes (virtual) or
+ // killed units (physical).
for (const VRegMaskOrUnit &Use : RegOpers.Uses) {
- if (!Use.VRegOrUnit.isVirtualReg())
- continue;
- Register Reg = Use.VRegOrUnit.asVirtualReg();
- LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
- if (LastUseMask.none())
- continue;
- // The LastUseMask is queried from the liveness information of instruction
- // which may be further down the schedule. Some lanes may actually not be
- // last uses for the current position.
- // FIXME: allow the caller to pass in the list of vreg uses that remain
- // to be bottom-scheduled to avoid searching uses at each query.
- LastUseMask =
- findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);
- if (LastUseMask.none())
- continue;
+ if (Use.VRegOrUnit.isVirtualReg()) {
+ Register Reg = Use.VRegOrUnit.asVirtualReg();
+ LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
+ if (LastUseMask.none())
+ continue;
+ // The LastUseMask is queried from the liveness information of instruction
+ // which may be further down the schedule. Some lanes may actually not be
+ // last uses for the current position.
+ // FIXME: allow the caller to pass in the list of vreg uses that remain
+ // to be bottom-scheduled to avoid searching uses at each query.
+ LastUseMask =
+ findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);
+ if (LastUseMask.none())
+ continue;
- auto It = VirtLiveRegs.find(Reg);
- LaneBitmask LiveMask =
- It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
- LaneBitmask NewMask = LiveMask & ~LastUseMask;
- PostUseMask[Reg] = NewMask;
- TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ auto It = VirtLiveRegs.find(Reg);
+ LaneBitmask LiveMask =
+ It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
+ LaneBitmask NewMask = LiveMask & ~LastUseMask;
+ PostUseMask[Reg] = NewMask;
+ TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ } else if (TrackPhysRegs) {
+ MCRegUnit Unit = Use.VRegOrUnit.asMCRegUnit();
+ unsigned U = static_cast<unsigned>(Unit);
+ if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SlotIdx))
+ TempPressure.dec(Unit, *SRI);
+ }
}
- // Generate liveness for defs.
+ // Process defs: increment pressure for new lanes (virtual) or
+ // new units (physical).
for (const VRegMaskOrUnit &Def : RegOpers.Defs) {
- if (!Def.VRegOrUnit.isVirtualReg())
- continue;
- Register Reg = Def.VRegOrUnit.asVirtualReg();
- auto PostIt = PostUseMask.find(Reg);
- LaneBitmask LiveMask;
- if (PostIt != PostUseMask.end()) {
- LiveMask = PostIt->second;
- } else {
- auto It = VirtLiveRegs.find(Reg);
- LiveMask = It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
- }
+ if (Def.VRegOrUnit.isVirtualReg()) {
+ Register Reg = Def.VRegOrUnit.asVirtualReg();
+ auto PostIt = PostUseMask.find(Reg);
+ LaneBitmask LiveMask;
+ if (PostIt != PostUseMask.end()) {
+ LiveMask = PostIt->second;
+ } else {
+ auto It = VirtLiveRegs.find(Reg);
+ LiveMask = It != VirtLiveRegs.end() ? It->second : LaneBitmask(0);
+ }
- LaneBitmask NewMask = LiveMask | Def.LaneMask;
- TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ LaneBitmask NewMask = LiveMask | Def.LaneMask;
+ TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
+ } else if (TrackPhysRegs) {
+ MCRegUnit Unit = Def.VRegOrUnit.asMCRegUnit();
+ unsigned U = static_cast<unsigned>(Unit);
+ if (!PhysLiveRegUnits.test(U))
+ TempPressure.inc(Unit, *SRI);
+ }
}
return TempPressure;
@@ -902,14 +1080,15 @@ bool GCNUpwardRPTracker::isValid() const {
dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
" LIS reported livesets mismatch:\n"
<< print(LISLR, *MRI);
- reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());
+ reportMismatch(LISLR, TrackedLR, SRI);
return false;
}
- auto LISPressure = getVirtRegPressure(*MRI, LISLR);
- if (LISPressure != CurPressure) {
+ auto ComputedPressure =
+ getVirtRegPressure(*MRI, LISLR) + constructPhysRegPressure();
+ if (ComputedPressure != CurPressure) {
dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
- << print(CurPressure) << "LIS rpt: " << print(LISPressure);
+ << print(CurPressure) << "Computed rpt: " << print(ComputedPressure);
return false;
}
return true;
@@ -1017,9 +1196,18 @@ bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
if (MBB.empty()) {
VirtLiveIn = VirtLiveOut = getVirtLiveRegs(MBBStartSlot, LIS, MRI);
RPAtMBBEnd = getVirtRegPressure(MRI, VirtLiveIn);
+ const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
+ BitVector SeenUnits(SRI->getNumRegUnits());
+ for (const auto &LI : MBB.liveins())
+ if (MRI.isAllocatable(LI.PhysReg))
+ for (MCRegUnit Unit : SRI->regunits(LI.PhysReg))
+ if (!SeenUnits.test(static_cast<unsigned>(Unit))) {
+ SeenUnits.set(static_cast<unsigned>(Unit));
+ RPAtMBBEnd.inc(Unit, *SRI);
+ }
} else {
- GCNDownwardRPTracker RPT(LIS);
- RPT.reset(MBB.front(), MBB.end());
+ GCNDownwardRPTracker RPT(LIS, MRI);
+ RPT.reset(MBB.front(), MBB.end(), /*VirtLiveRegs=*/nullptr, &MBB);
VirtLiveIn = RPT.getVirtLiveRegs();
@@ -1033,8 +1221,8 @@ bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
RPAtMBBEnd = RPT.getPressure();
}
} else {
- GCNUpwardRPTracker RPT(LIS);
- RPT.reset(MRI, MBBLastSlot);
+ GCNUpwardRPTracker RPT(LIS, MRI);
+ RPT.reset(MRI, MBBLastSlot, &MBB);
VirtLiveOut = RPT.getVirtLiveRegs();
RPAtMBBEnd = RPT.getPressure();
@@ -1105,9 +1293,10 @@ LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,
unsigned MaxNumRegs = 0;
const MachineInstr *MaxPressureMI = nullptr;
- GCNUpwardRPTracker RPT(LIS);
+ GCNUpwardRPTracker RPT(LIS, MRI);
for (const MachineBasicBlock &MBB : MF) {
- RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
+ RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot(),
+ &MBB);
for (const MachineInstr &MI : reverse(MBB)) {
RPT.recede(MI);
unsigned NumRegs = RPT.getMaxPressure().getNumRegs(Kind);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 1c09042a5784c..92819b15eaaf2 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -18,6 +18,7 @@
#define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
#include "GCNSubtarget.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/RegisterPressure.h"
#include <algorithm>
@@ -125,11 +126,28 @@ struct GCNRegPressure {
return std::max(UnifiedSpill, ArchSpill + AGPRSpill);
}
+ /// Adjust pressure for a virtual register.
void inc(unsigned Reg,
LaneBitmask PrevMask,
LaneBitmask NewMask,
const MachineRegisterInfo &MRI);
+ /// Increment pressure for a physical register unit.
+ void inc(MCRegUnit Unit, const SIRegisterInfo &SRI) {
+ adjustPhysUnitPressure(Unit, /*IsAdd=*/true, SRI);
+ }
+
+ /// Decrement pressure for a physical register unit.
+ void dec(MCRegUnit Unit, const SIRegisterInfo &SRI) {
+ adjustPhysUnitPressure(Unit, /*IsAdd=*/false, SRI);
+ }
+
+private:
+ static unsigned pressureSetToRegKind(unsigned PSetID);
+ void adjustPhysUnitPressure(MCRegUnit Unit, bool IsAdd,
+ const SIRegisterInfo &SRI);
+
+public:
bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure &O,
unsigned DynamicVGPRBlockSize) const {
return getOccupancy(ST, DynamicVGPRBlockSize) >
@@ -324,12 +342,30 @@ class GCNRPTracker {
protected:
const LiveIntervals &LIS;
+ mutable const MachineRegisterInfo *MRI = nullptr;
+ const SIRegisterInfo *SRI = nullptr;
+
LiveRegSet VirtLiveRegs;
+
+ // Physical register liveness tracked at the register-unit level.
+ // Each bit corresponds to a register unit. This avoids aliasing issues
+ // since overlapping physical registers share the same underlying units.
+ BitVector PhysLiveRegUnits;
+
GCNRegPressure CurPressure, MaxPressure;
+
+ // Flag to control whether physical register tracking is active.
+ // Set to true when GCNTrackers are enabled, false otherwise.
+ bool TrackPhysRegs = false;
+
const MachineInstr *LastTrackedMI = nullptr;
- mutable const MachineRegisterInfo *MRI = nullptr;
- GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
+ GCNRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : LIS(LIS), MRI(&MRI),
+ SRI(static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo())),
+ PhysLiveRegUnits(SRI->getNumRegUnits()) {
+ updatePhysRegTracking();
+ }
/// Resets tracker before or \p After the provided \p MI, which can be a debug
/// instruction.
@@ -346,14 +382,60 @@ class GCNRPTracker {
LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;
+ bool shouldTrackPhysReg(Register Reg) const {
+ return TrackPhysRegs && Reg.isPhysical() && MRI->isAllocatable(Reg);
+ }
+
+ // Check if a register unit is live at a given slot index per LIS.
+ bool isUnitLiveAt(MCRegUnit Unit, SlotIndex SI) const;
+
+ // Construct physical register pressure from PhysLiveRegUnits.
+ GCNRegPressure constructPhysRegPressure() const;
+
+ // Add units of Reg that are not already live. Increases Pressure for each
+ // newly live unit.
+ void addUnitsAndIncPressure(MCRegister Reg, GCNRegPressure &Pressure);
+
+ // Remove all live units of Reg. Decreases Pressure for each removed unit.
+ void removeUnitsAndDecPressure(MCRegister Reg, GCNRegPressure &Pressure);
+
+ // Remove units of Reg that are currently live but killed at SI.
+ // Decreases Pressure for each killed unit.
+ void removeKilledUnitsAndDecPressure(MCRegister Reg, SlotIndex SI,
+ GCNRegPressure &Pressure);
+
public:
+ // Enable physical register tracking only if both GCNTrackers and
+ // TrackPhysRegInTrackers are true.
+ void updatePhysRegTracking();
+
/// Resets tracker with the provided \p VirtLiveRegs.
void reset(const MachineRegisterInfo &MRI, const LiveRegSet &VirtLiveRegs);
+ // Reset tracker with both virtual and physical live register state.
+ void reset(const MachineRegisterInfo &MRInfo,
+ const LiveRegSet &VirtLiveRegsSet, const BitVector &PhysLiveUnits);
+
// live regs for the current state
const decltype(VirtLiveRegs) &getVirtLiveRegs() const { return VirtLiveRegs; }
+ const BitVector &getPhysLiveRegUnits() const { return PhysLiveRegUnits; }
const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
+ /// Initialize PhysLiveRegUnits from a range of RegisterMaskPair entries
+ /// and update CurPressure/MaxPressure accordingly.
+ template <typename RangeT>
+ void initPhysLiveUnitsFromRegMaskPairs(RangeT &&Pairs) {
+ assert(TrackPhysRegs && "physical register tracking must be enabled");
+ for (const auto &RM : Pairs)
+ if (MRI->isAllocatable(RM.PhysReg))
+ addUnitsAndIncPressure(RM.PhysReg, CurPressure);
+ MaxPressure = max(MaxPressure, CurPressure);
+ }
+
+ /// Restore PhysLiveRegUnits from a previously saved BitVector and update
+ /// CurPressure/MaxPressure accordingly.
+ void initPhysLiveUnits(const BitVector &PhysLiveUnits);
+
void clearMaxPressure() { MaxPressure.clear(); }
GCNRegPressure getPressure() const { return CurPressure; }
@@ -371,13 +453,36 @@ getVirtLiveRegs(SlotIndex SI, const LiveIntervals &LIS,
class GCNUpwardRPTracker : public GCNRPTracker {
public:
- GCNUpwardRPTracker(const LiveIntervals &LIS) : GCNRPTracker(LIS) {}
+ GCNUpwardRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : GCNRPTracker(LIS, MRI) {}
using GCNRPTracker::reset;
- /// Resets tracker to the point just after \p MI (in program order), which can
- /// be a debug instruction.
- void reset(const MachineInstr &MI) { reset(MI, /*After=*/true); }
+ /// reset tracker at the specified slot index \p SI. If \p SeedPhysMBB is
+ /// non-null, also seed physical live-out state from that MBB's successors.
+ void reset(const MachineRegisterInfo &MRI, SlotIndex SI,
+ const MachineBasicBlock *SeedPhysMBB = nullptr) {
+ GCNRPTracker::reset(MRI, llvm::getVirtLiveRegs(SI, LIS, MRI));
+ if (SeedPhysMBB && TrackPhysRegs &&
+ SeedPhysMBB->getParent()->getProperties().hasTracksLiveness())
+ initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveouts());
+ }
+
+ /// reset tracker to the end of the \p MBB and seed physical live-outs
+ /// from the MBB's successors.
+ void reset(const MachineBasicBlock &MBB) {
+ SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(&MBB);
+ reset(MBB.getParent()->getRegInfo(), MBBLastSlot, &MBB);
+ }
+
+ /// reset tracker to the point just after \p MI (in program order).
+ /// If \p SeedPhysMBB is non-null, also seed physical live-out state from
+ /// that MBB's successors.
+ void reset(const MachineInstr &MI,
+ const MachineBasicBlock *SeedPhysMBB = nullptr) {
+ reset(MI.getMF()->getRegInfo(), LIS.getInstructionIndex(MI).getDeadSlot(),
+ SeedPhysMBB);
+ }
/// Move to the state of RP just before the \p MI . If \p UseInternalIterator
/// is set, also update the internal iterators. Setting \p UseInternalIterator
@@ -409,7 +514,8 @@ class GCNDownwardRPTracker : public GCNRPTracker {
MachineBasicBlock::const_iterator MBBEnd;
public:
- GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
+ GCNDownwardRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
+ : GCNRPTracker(LIS, MRI) {}
using GCNRPTracker::reset;
@@ -424,10 +530,18 @@ class GCNDownwardRPTracker : public GCNRPTracker {
/// Reset tracker to the point before the \p MI filling \p VirtLiveRegs upon
/// this point using LIS. \p End must be between the MI and the end of its
- /// parent block (inclusive). \p returns false if the range [MI, End) is empty
- /// except debug values.
+ /// parent block (inclusive). If \p SeedPhysMBB is non-null, also seed
+ /// physical live-in state from that MBB's live-in list. \p returns false if
+ /// the range [MI, End) is empty except debug values.
bool reset(const MachineInstr &MI, MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegs = nullptr);
+ const LiveRegSet *VirtLiveRegs = nullptr,
+ const MachineBasicBlock *SeedPhysMBB = nullptr);
+
+ /// Reset tracker to the point before \p MI, restoring both virtual and
+ /// physical register state from saved snapshots.
+ /// \p returns false if block is empty except debug values.
+ bool reset(const MachineInstr &MI, const LiveRegSet &VirtLiveRegs,
+ const BitVector &PhysLiveUnits);
/// Move to the state right before the next MI or after the end of MBB.
/// \p returns false if reached end of the block.
@@ -465,11 +579,13 @@ class GCNDownwardRPTracker : public GCNRPTracker {
/// Reset tracker to \p Begin (filling \p LiveRegs upon this point using LIS)
/// and advance to \p End, which must be between \p Begin and the end of its
- /// parent block (inclusive). \p returns false if the range [Begin, End) is
- /// empty except debug values.
+ /// parent block (inclusive). If \p PhysLiveUnits is non-null, the physical
+ /// register state is restored from that snapshot after the reset. \p returns
+ /// false if the range [Begin, End) is empty except debug values.
bool advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
- const LiveRegSet *VirtLiveRegsCopy = nullptr);
+ const LiveRegSet *VirtLiveRegsCopy = nullptr,
+ const BitVector *PhysLiveUnits = nullptr);
/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
/// Calculate the impact \p MI will have on CurPressure and \return the
@@ -557,6 +673,11 @@ GCNRegPressure getVirtRegPressure(const MachineRegisterInfo &MRI,
return Res;
}
+/// Adds to \p RP the pressure contributed by the live physical register units
+/// set in \p PhysLiveUnits.
+void addPhysRegPressure(GCNRegPressure &RP, const SIRegisterInfo &SRI,
+ const BitVector &PhysLiveUnits);
+
bool isEqual(const GCNRPTracker::LiveRegSet &S1,
const GCNRPTracker::LiveRegSet &S2);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index bb52820671725..df918e7d35a4e 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -79,6 +79,12 @@ static cl::opt<bool> GCNTrackers(
cl::desc("Use the AMDGPU specific RPTrackers during scheduling"),
cl::init(false));
+static cl::opt<bool> TrackPhysRegInTrackers(
+ "amdgpu-trackers-physical-register-tracking", cl::Hidden,
+ cl::desc("When using GCN trackers, count physical registers (e.g. from "
+ "inline asm) in pressure."),
+ cl::init(true));
+
static cl::opt<unsigned> PendingQueueLimit(
"amdgpu-scheduler-pending-queue-limit", cl::Hidden,
cl::desc(
@@ -135,7 +141,8 @@ const unsigned ScheduleMetrics::ScaleFactor = 100;
GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
: GenericScheduler(C), TargetOccupancy(0), MF(nullptr),
- DownwardTracker(*C->LIS), UpwardTracker(*C->LIS), HasHighPressure(false) {
+ DownwardTracker(*C->LIS, C->MF->getRegInfo()),
+ UpwardTracker(*C->LIS, C->MF->getRegInfo()), HasHighPressure(false) {
if (GCNTrackers.getNumOccurrences() > 0)
GCNTrackersOverride = GCNTrackers;
}
@@ -144,7 +151,6 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
GenericScheduler::initialize(DAG);
MF = &DAG->MF;
-
const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
SGPRExcessLimit =
@@ -209,6 +215,14 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
<< ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
}
+void GCNRPTracker::updatePhysRegTracking() {
+ if (!GCNTrackers || !TrackPhysRegInTrackers) {
+ TrackPhysRegs = false;
+ return;
+ }
+ TrackPhysRegs = true;
+}
+
/// Checks whether \p SU can use the cached DAG pressure diffs to compute the
/// current register pressure.
///
@@ -1068,11 +1082,19 @@ void GCNScheduleDAGMILive::schedule() {
GCNRegPressure
GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const {
- if (Regions[RegionIdx].first == Regions[RegionIdx].second)
- return llvm::getVirtRegPressure(MRI, VirtLiveIns[RegionIdx]);
- GCNDownwardRPTracker RPTracker(*LIS);
+ if (Regions[RegionIdx].first == Regions[RegionIdx].second) {
+ GCNRegPressure RP = llvm::getVirtRegPressure(MRI, VirtLiveIns[RegionIdx]);
+ // Fold in physical live-in pressure so that empty regions are consistent
+ // with non-empty regions. This is a no-op when physical tracking is off,
+ // since the snapshot has no set units.
+ const auto *SRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ llvm::addPhysRegPressure(RP, *SRI, PhysLiveIns[RegionIdx]);
+ return RP;
+ }
+ GCNDownwardRPTracker RPTracker(*LIS, MF.getRegInfo());
RPTracker.advance(Regions[RegionIdx].first, Regions[RegionIdx].second,
- &VirtLiveIns[RegionIdx]);
+ &VirtLiveIns[RegionIdx], &PhysLiveIns[RegionIdx]);
return RPTracker.moveMaxPressure();
}
@@ -1084,7 +1106,7 @@ static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin,
void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
const MachineBasicBlock *MBB) {
- GCNDownwardRPTracker RPTracker(*LIS);
+ GCNDownwardRPTracker RPTracker(*LIS, MF.getRegInfo());
// If the block has the only successor then live-ins of that successor are
// live-outs of the current block. We can reuse calculated live set if the
@@ -1117,7 +1139,7 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second);
if (VirtLiveInIt != MBBVirtLiveIns.end()) {
auto VirtLiveIn = std::move(VirtLiveInIt->second);
- RPTracker.reset(*MBB->begin(), MBB->end(), &VirtLiveIn);
+ RPTracker.reset(*MBB->begin(), MBB->end(), &VirtLiveIn, MBB);
MBBVirtLiveIns.erase(VirtLiveInIt);
} else {
I = Rgn.first;
@@ -1125,7 +1147,7 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
#ifdef EXPENSIVE_CHECKS
assert(isEqual(getVirtLiveRegsBefore(*NonDbgMI, *LIS), VirtLiveInSet));
#endif
- RPTracker.reset(*I, I->getParent()->end(), &VirtLiveInSet);
+ RPTracker.reset(*I, I->getParent()->end(), &VirtLiveInSet, MBB);
}
for (;;) {
@@ -1133,10 +1155,12 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
if (Regions[CurRegion].first == I || NonDbgMI == I) {
VirtLiveIns[CurRegion] = RPTracker.getVirtLiveRegs();
+ PhysLiveIns[CurRegion] = RPTracker.getPhysLiveRegUnits();
RPTracker.clearMaxPressure();
}
if (Regions[CurRegion].second == I) {
+ PhysLiveOuts[CurRegion] = RPTracker.getPhysLiveRegUnits();
Pressure[CurRegion] = RPTracker.moveMaxPressure();
if (CurRegion-- == RegionIdx)
break;
@@ -1204,6 +1228,8 @@ void GCNScheduleDAGMILive::finalizeSchedule() {
// MachineScheduler after all regions have been recorded by
// GCNScheduleDAGMILive::schedule().
VirtLiveIns.resize(Regions.size());
+ PhysLiveIns.resize(Regions.size());
+ PhysLiveOuts.resize(Regions.size());
Pressure.resize(Regions.size());
RegionsWithHighRP.resize(Regions.size());
RegionsWithExcessRP.resize(Regions.size());
@@ -1250,9 +1276,11 @@ void GCNScheduleDAGMILive::runSchedStages() {
if (S.useGCNTrackers()) {
const unsigned RegionIdx = Stage->getRegionIdx();
- S.getDownwardTracker()->reset(MRI, VirtLiveIns[RegionIdx]);
+ S.getDownwardTracker()->reset(MRI, VirtLiveIns[RegionIdx],
+ PhysLiveIns[RegionIdx]);
S.getUpwardTracker()->reset(
- MRI, RegionVirtLiveOuts.getVirtLiveRegsForRegionIdx(RegionIdx));
+ MRI, RegionVirtLiveOuts.getVirtLiveRegsForRegionIdx(RegionIdx),
+ PhysLiveOuts[RegionIdx]);
}
ScheduleDAGMILive::schedule();
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index be459ed36d6d8..19dee8b870a82 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -293,6 +293,12 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
// Region live-in cache.
SmallVector<GCNRPTracker::LiveRegSet, 32> VirtLiveIns;
+ // Per-region physical register live-in cache (register unit BitVectors).
+ SmallVector<BitVector, 32> PhysLiveIns;
+
+ // Per-region physical register live-out cache (register unit BitVectors).
+ SmallVector<BitVector, 32> PhysLiveOuts;
+
// Region pressure cache.
SmallVector<GCNRegPressure, 32> Pressure;
diff --git a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
index 45f0e6a8ba1e7..f121fc1b5f206 100644
--- a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
@@ -275,7 +275,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
"amdgpu-max-memory-clause", MaxClause);
for (MachineBasicBlock &MBB : MF) {
- GCNDownwardRPTracker RPT(*LIS);
+ GCNDownwardRPTracker RPT(*LIS, *MRI);
MachineBasicBlock::instr_iterator Next;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
MachineInstr &MI = *I;
@@ -290,16 +290,17 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
continue;
if (!RPT.getNext().isValid())
- RPT.reset(MI, MBB.end());
+ RPT.reset(MI, MBB.end(), /*VirtLiveRegs=*/nullptr, &MBB);
else { // Advance the state to the current MI.
RPT.advance(MachineBasicBlock::const_iterator(MI));
RPT.advanceBeforeNext();
}
const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getVirtLiveRegs());
+ const BitVector PhysRegsCopy(RPT.getPhysLiveRegUnits());
RegUse Defs, Uses;
if (!processRegUses(MI, Defs, Uses, RPT)) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
continue;
}
@@ -323,7 +324,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
++Length;
}
if (Length < 2) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
continue;
}
@@ -391,7 +392,7 @@ bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
}
// Restore the state after processing the end of the bundle.
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ RPT.reset(MI, LiveRegsCopy, PhysRegsCopy);
if (!Kill)
continue;
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index 8c9e4a5a26c83..055d6bfd83f95 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -11144,8 +11144,6 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-GCNTRACKERS-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
; GFX908-GCNTRACKERS-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -11205,6 +11203,8 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF27]], implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[DEF28:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr8, implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 $vgpr9, implicit $exec, implicit $mode
@@ -11445,8 +11445,6 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-GCNTRACKERS-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
; GFX908-GCNTRACKERS-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -11508,18 +11506,20 @@ body: |
; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF28]], implicit $exec, implicit $mode
; GFX908-GCNTRACKERS-NEXT: [[DEF29:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: [[DEF30:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF29]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 [[DEF30]], implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX908-GCNTRACKERS-NEXT: $vgpr9 = IMPLICIT_DEF
; GFX908-GCNTRACKERS-NEXT: S_BRANCH %bb.1
; GFX908-GCNTRACKERS-NEXT: {{ $}}
; GFX908-GCNTRACKERS-NEXT: bb.1:
; GFX908-GCNTRACKERS-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 255
; GFX908-GCNTRACKERS-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 [[S_MOV_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_1]], implicit [[V_CVT_I32_F32_e32_9]]
- ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
- ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF29]]
- ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_28]], implicit [[V_CVT_I32_F32_e32_30]], implicit [[DEF30]]
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 1, implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_30]], implicit [[V_CVT_I32_F32_e32_28]], implicit [[DEF29]]
+ ; GFX908-GCNTRACKERS-NEXT: [[V_CVT_I32_F32_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F32_e32 2, implicit $exec, implicit $mode
+ ; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_31]], implicit [[V_CVT_I32_F32_e32_29]], implicit [[DEF30]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_2]], implicit [[V_CVT_I32_F32_e32_10]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_3]], implicit [[V_CVT_I32_F32_e32_11]]
; GFX908-GCNTRACKERS-NEXT: S_NOP 0, implicit [[V_CVT_I32_F32_e32_4]], implicit [[V_CVT_I32_F32_e32_12]]
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index 67065a6db8028..a76318434a391 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -1,12 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -mtriple=amdgpu8.10-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX900 %s
-; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX942 %s
-; RUN: llc -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10_1 %s
-; RUN: llc -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefix=GFX10_3 %s
-; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX942 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10_1 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10_3 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX7-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_1-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_3-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX11-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX12-GCNTRACKERS %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
@@ -564,6 +572,541 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX7-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s32, 6
+; GFX7-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX7-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0x4040
+; GFX7-GCNTRACKERS-NEXT: v_mad_u32_u24 v0, v0, 64, s32
+; GFX7-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 6, v0
+; GFX7-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX8-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, 64, v0
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX8-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0x4040
+; GFX8-GCNTRACKERS-NEXT: v_mad_u32_u24 v0, v0, 64, s32
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 6, v0
+; GFX8-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 64, v0
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 0x4040, v0
+; GFX900-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v0
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x101100
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4044
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v23, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX942-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 0x4040
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4044
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v23, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v24, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v24, 0x4040, v24
+; GFX10_1-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v24
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v24, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v24, 0x4040, v24
+; GFX10_3-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v24
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80880
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4044
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v23, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 0x4040
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4044
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v23, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v23, s32 offset:16388 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX12-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s32
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: s_add_co_ci_u32 s59, s32, 0x4000
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX12-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v23, off, s32 offset:16388 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
%alloca1 = alloca i32, align 4, addrspace(5)
call void asm sideeffect "; use alloca0 $0", "v"(ptr addrspace(5) %alloca0)
@@ -1085,6 +1628,485 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_mad_u32_u24 v22, 16, 64, s32
+; GFX7-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v22, 6, v22
+; GFX7-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_mad_u32_u24 v22, 16, 64, s32
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v22, 6, v22
+; GFX8-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 6, s32
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v22, 16, v22
+; GFX900-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x100400
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4010
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v21, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 16
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x4010
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v21, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v22, 16, v22
+; GFX10_1-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v21, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v22, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v22, 16, v22
+; GFX10_3-GCNTRACKERS-NEXT: v_readfirstlane_b32 s54, v22
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x80200
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4010
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v21, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: s_addc_u32 s59, s32, 16
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GCNTRACKERS-NEXT: s_bitcmp1_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_bitset0_b32 s59, 0
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s59
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x4010
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v21, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowest_offset:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v21, s32 offset:16384 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v21, s31, 16
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s59, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s32
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v21, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v21, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v21, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v21, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v21, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v21, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v21, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v21, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v21, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v21, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v21, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v21, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v21, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v21, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v21, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v21, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v21, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v21, off, s32 offset:16384 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 16, addrspace(5)
; Force no SGPRs to be available for the carry-out of the vector add.
@@ -1661,6 +2683,543 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
; GFX12-NEXT: s_mov_b32 exec_lo, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX7-GCNTRACKERS: ; %bb.0:
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX7-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s28, 17
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s29, 18
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX7-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX7-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s32, 6
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX7-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX7-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX7-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX7-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX7-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX7-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s28, v23, 17
+; GFX7-GCNTRACKERS-NEXT: v_readlane_b32 s29, v23, 18
+; GFX7-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX7-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX7-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX7-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX8-GCNTRACKERS: ; %bb.0:
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX8-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX8-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX8-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX8-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX8-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, 64, v0
+; GFX8-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX8-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX8-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX8-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX8-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX8-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX8-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX8-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX8-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX900-GCNTRACKERS: ; %bb.0:
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v23, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX900-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s33, 0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s34, 1
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s35, 2
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s36, 3
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s37, 4
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s38, 5
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s39, 6
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s48, 7
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s49, 8
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s50, 9
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s51, 10
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s52, 11
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s53, 12
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s54, 13
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s55, 14
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s30, 15
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v23, s31, 16
+; GFX900-GCNTRACKERS-NEXT: s_lshr_b32 s5, s32, 6
+; GFX900-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s4, s5, 0x4240
+; GFX900-GCNTRACKERS-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; GFX900-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 64, v0
+; GFX900-GCNTRACKERS-NEXT: v_writelane_b32 v22, s4, 0
+; GFX900-GCNTRACKERS-NEXT: s_and_b64 s[4:5], 0, exec
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 0
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX900-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX900-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s30, v23, 15
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s31, v23, 16
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s55, v23, 14
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s54, v23, 13
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s53, v23, 12
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s52, v23, 11
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s51, v23, 10
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s50, v23, 9
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s49, v23, 8
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s48, v23, 7
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s39, v23, 6
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s38, v23, 5
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s37, v23, 4
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s36, v23, 3
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s35, v23, 2
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s34, v23, 1
+; GFX900-GCNTRACKERS-NEXT: v_readlane_b32 s33, v23, 0
+; GFX900-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201000
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_add_i32 s6, s32, 0x201100
+; GFX900-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s6 ; 4-byte Folded Reload
+; GFX900-GCNTRACKERS-NEXT: s_mov_b64 exec, s[4:5]
+; GFX900-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX900-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX942-GCNTRACKERS: ; %bb.0:
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x8040
+; GFX942-GCNTRACKERS-NEXT: scratch_store_dword off, v22, s2 ; 4-byte Folded Spill
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX942-GCNTRACKERS-NEXT: s_nop 1
+; GFX942-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX942-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s58, s32, 0x4240
+; GFX942-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX942-GCNTRACKERS-NEXT: s_and_b64 s[60:61], 0, exec
+; GFX942-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX942-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX942-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX942-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX942-GCNTRACKERS-NEXT: s_xor_saveexec_b64 s[0:1], -1
+; GFX942-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x8040
+; GFX942-GCNTRACKERS-NEXT: scratch_load_dword v22, off, s2 ; 4-byte Folded Reload
+; GFX942-GCNTRACKERS-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_1-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX10_1-GCNTRACKERS: ; %bb.0:
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_1-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_1-GCNTRACKERS-NEXT: s_lshr_b32 s4, s32, 5
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s58, s4, 0x4240
+; GFX10_1-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_1-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_1-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX10_1-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX10_1-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX10_1-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_1-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_1-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_1-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_1-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10_3-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX10_3-GCNTRACKERS: ; %bb.0:
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_3-GCNTRACKERS-NEXT: buffer_store_dword v22, off, s[0:3], s5 ; 4-byte Folded Spill
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 5, s32
+; GFX10_3-GCNTRACKERS-NEXT: s_lshr_b32 s4, s32, 5
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s58, s4, 0x4240
+; GFX10_3-GCNTRACKERS-NEXT: s_and_b32 s4, 0, exec_lo
+; GFX10_3-GCNTRACKERS-NEXT: v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX10_3-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX10_3-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX10_3-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX10_3-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s4, -1
+; GFX10_3-GCNTRACKERS-NEXT: s_add_i32 s5, s32, 0x100800
+; GFX10_3-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s5 ; 4-byte Folded Reload
+; GFX10_3-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s4
+; GFX10_3-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX10_3-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX11-GCNTRACKERS: ; %bb.0:
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x8040
+; GFX11-GCNTRACKERS-NEXT: scratch_store_b32 off, v22, s1 ; 4-byte Folded Spill
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX11-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s58, s32, 0x4240
+; GFX11-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX11-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX11-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX11-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX11-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x8040
+; GFX11-GCNTRACKERS-NEXT: scratch_load_b32 v22, off, s1 ; 4-byte Folded Reload
+; GFX11-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GCNTRACKERS-LABEL: scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_immoffset:
+; GFX12-GCNTRACKERS: ; %bb.0:
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_expcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_samplecnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_wait_kmcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_store_b32 off, v22, s32 offset:32768 ; 4-byte Folded Spill
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s33, 0
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s34, 1
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s35, 2
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s36, 3
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s37, 4
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s38, 5
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s39, 6
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s48, 7
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s49, 8
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s50, 9
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s51, 10
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s52, 11
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s53, 12
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s54, 13
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s55, 14
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s30, 15
+; GFX12-GCNTRACKERS-NEXT: v_writelane_b32 v22, s31, 16
+; GFX12-GCNTRACKERS-NEXT: s_add_co_i32 s58, s32, 0x4200
+; GFX12-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s32
+; GFX12-GCNTRACKERS-NEXT: s_and_b32 s0, 0, exec_lo
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use alloca0 v0
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: ; kill: def $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 killed $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 def $sgpr54
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 s54, s58
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMSTART
+; GFX12-GCNTRACKERS-NEXT: ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s54, scc
+; GFX12-GCNTRACKERS-NEXT: ;;#ASMEND
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s30, v22, 15
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s31, v22, 16
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s55, v22, 14
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s54, v22, 13
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s53, v22, 12
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s52, v22, 11
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s51, v22, 10
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s50, v22, 9
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s49, v22, 8
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s48, v22, 7
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s39, v22, 6
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s38, v22, 5
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s37, v22, 4
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s36, v22, 3
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s35, v22, 2
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s34, v22, 1
+; GFX12-GCNTRACKERS-NEXT: v_readlane_b32 s33, v22, 0
+; GFX12-GCNTRACKERS-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX12-GCNTRACKERS-NEXT: scratch_load_b32 v22, off, s32 offset:32768 ; 4-byte Folded Reload
+; GFX12-GCNTRACKERS-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GCNTRACKERS-NEXT: s_wait_loadcnt 0x0
+; GFX12-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31]
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
%alloca1 = alloca [4096 x i32], align 4, addrspace(5)
call void asm sideeffect "; use alloca0 $0", "v"(ptr addrspace(5) %alloca0)
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
new file mode 100644
index 0000000000000..516db40a5af76
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
@@ -0,0 +1,43 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+
+# Test that the upward register pressure tracker accounts for early-clobber
+# physical register defs overlapping with physical register uses.
+# With physreg tracking, the EC def s[10:11] (2 SGPRs) overlaps with the use
+# s12 (1 SGPR), producing higher max pressure at the INLINEASM instruction.
+
+---
+name: ec_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: ec_physreg
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ; RPU: 3 1
+ ; RPU: 5 1 INLINEASM &"s_mov_b64 $0, $1" {{.*}} early-clobber $sgpr10_sgpr11
+ ; RPU: 4 1
+ ;
+ ; RPU-NOPHYS-LABEL: name: ec_physreg
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, $1" {{.*}} early-clobber $sgpr10_sgpr11
+ ; RPU-NOPHYS: 2 1
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ INLINEASM &"s_mov_b32 $0, 42", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr12
+ %3:sreg_32 = COPY $sgpr12
+ $sgpr12 = COPY %3
+ INLINEASM &"s_mov_b64 $0, $1", 1 /* sideeffect attdialect */, 11 /* regdef-ec */, implicit-def early-clobber $sgpr10_sgpr11, 9 /* reguse */, $sgpr12
+ %4:sreg_64 = COPY $sgpr10_sgpr11
+ %5:sreg_32 = COPY %4.sub0
+ %6:vgpr_32 = COPY %5
+ GLOBAL_STORE_DWORD_SADDR %2, %6, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
new file mode 100644
index 0000000000000..bc6f1e1888bbc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
@@ -0,0 +1,239 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s 2>&1 | FileCheck %s --check-prefix=RPU
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward %s 2>&1 | FileCheck %s --check-prefix=RPD
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
+
+# Tests for physical register pressure tracking edge cases:
+# 1. Non-allocatable registers (e.g. $scc) should not affect pressure.
+# 2. Aliasing: a tuple def ($sgpr10_sgpr11) does not kill a live sub-register
+# ($sgpr11) tracked under a different name, leading to over-counted pressure.
+
+# Non-allocatable physical register should not change pressure.
+---
+name: nonallocatable_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: nonallocatable_physreg
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ; RPU: 2 1
+ ; RPU: 2 1 S_NOP 0, implicit-def $scc
+ ; RPU: 2 1
+ ;
+ ; RPU-NOPHYS-LABEL: name: nonallocatable_physreg
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 S_NOP 0, implicit-def $scc
+ ; RPU-NOPHYS: 2 1
+ ;
+ ; RPD-LABEL: name: nonallocatable_physreg
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Initial pressure includes $sgpr8_sgpr9 live-in (2 SGPR units).
+ ;
+ ; RPD: 2 0
+ ; RPD: 2 1
+ ; RPD: 2 1 S_NOP 0, implicit-def $scc
+ ; RPD: 2 1
+ ;
+ ; RPD-NOPHYS-LABEL: name: nonallocatable_physreg
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-in physical register pressure.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 2 1 S_NOP 0, implicit-def $scc
+ ; RPD-NOPHYS: 2 1
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ S_NOP 0, implicit-def $scc
+ GLOBAL_STORE_DWORD_SADDR %2, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
+
+# Aliasing: $sgpr10_sgpr11 is defined, then $sgpr11 (a sub-register) is used.
+# Unit-level tracking correctly handles this: the upward tracker adds
+# $sgpr11's unit when processing the use, and when receding past the def of
+# $sgpr10_sgpr11, it finds and removes $sgpr11's unit, correctly decrementing
+# pressure.
+---
+name: aliased_physreg_tuple_def_subreg_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ;
+ ; The def of $sgpr10_sgpr11 correctly kills $sgpr11's unit via
+ ; unit-level tracking, reducing SGPR pressure from 3 to 2.
+ ;
+ ; RPU: 2 1
+ ; RPU: 3 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPU: 3 1
+ ; RPU: 3 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPU-NOPHYS-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no physical register pressure is counted,
+ ; so the INLINEASM shows no changes in pressure.
+ ;
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPU-NOPHYS: 2 1
+ ; RPU-NOPHYS: 3 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPD-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Initial pressure includes $sgpr8_sgpr9 live-in (2 SGPR units).
+ ; Downward tracker: the INLINEASM def adds $sgpr10_sgpr11 (2 units =
+ ; 2 SGPRs). At the COPY use of $sgpr11, the corresponding reg-unit
+ ; is not found in the cached live ranges, so it conservatively keeps
+ ; the unit live. The virtual register def leads to the increase in
+ ; pressure from 4 to 5.
+ ;
+ ; RPD: 2 0
+ ; RPD: 2 1
+ ; RPD: 4 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPD: 4 1
+ ; RPD: 5 1 %3:sreg_32 = COPY $sgpr11
+ ;
+ ; RPD-NOPHYS-LABEL: name: aliased_physreg_tuple_def_subreg_use
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-in physical register pressure
+ ; and no physical register pressure changes at instructions.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 2 1 INLINEASM &"s_mov_b64 $0, 0"{{.*}} implicit-def $sgpr10_sgpr11
+ ; RPD-NOPHYS: 2 1
+ ; RPD-NOPHYS: 3 1 %3:sreg_32 = COPY $sgpr11
+ bb.0:
+ liveins: $sgpr8_sgpr9
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ INLINEASM &"s_mov_b64 $0, 0", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr10_sgpr11
+ %3:sreg_32 = COPY $sgpr11
+ %4:vgpr_32 = COPY %3
+ GLOBAL_STORE_DWORD_SADDR %2, %4, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
+
+# Live-out seeding: bb.0 defines $sgpr10 via INLINEASM and branches to bb.1.
+# bb.1 has $sgpr10 as a live-in. The upward tracker for bb.0 is initialized
+# with bb.0's live-outs (= successor bb.1's live-ins). Without live-out
+# seeding, the upward tracker treats the INLINEASM def as dead and never
+# accounts for $sgpr10's pressure in the region below the def.
+---
+name: physreg_liveout
+tracksRegLiveness: true
+machineFunctionInfo:
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ ; RPU-LABEL: name: physreg_liveout
+ ; RPU: bb.0:
+ ; RPU: SGPR VGPR
+ ;
+ ; Top of bb.0: after receding past all instructions, $sgpr8_sgpr9 remains
+ ; live from the COPY use. SGPR=2 ($sgpr8_sgpr9 = 2 reg-units).
+ ;
+ ; RPU: 2 0
+ ; RPU: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPU: 2 0
+ ; RPU: 2 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPU: 2 0
+ ;
+ ; Below the INLINEASM def, $sgpr10 is live (from live-out seeding),
+ ; so SGPR=3 (%1 = 2 units + $sgpr10 = 1 unit).
+ ;
+ ; RPU: 3 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPU: 3 0
+ ; RPU: 3 0 S_BRANCH %bb.1
+ ; RPU: 3 0
+ ;
+ ; RPU-NOPHYS-LABEL: name: physreg_liveout
+ ; RPU-NOPHYS: bb.0:
+ ; RPU-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking, no live-out seeding: the INLINEASM def is dead
+ ; from the upward tracker's perspective, so $sgpr10 is never counted.
+ ; $sgpr8_sgpr9 is also not tracked, so top pressure is 0.
+ ;
+ ; RPU-NOPHYS: 0 0
+ ; RPU-NOPHYS: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPU-NOPHYS: 2 0
+ ; RPU-NOPHYS: 2 0 S_BRANCH %bb.1
+ ; RPU-NOPHYS: 2 0
+ ;
+ ; RPD-LABEL: name: physreg_liveout
+ ; RPD: bb.0:
+ ; RPD: SGPR VGPR
+ ;
+ ; Downward tracker: initial pressure includes $sgpr8_sgpr9 live-in (2).
+ ; INLINEASM adds $sgpr10 (1 unit), bringing pressure to 3.
+ ;
+ ; RPD: 2 0
+ ; RPD: 4 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPD: 2 0
+ ; RPD: 4 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPD: 2 0
+ ; RPD: 3 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPD: 3 0
+ ; RPD: 3 0 S_BRANCH %bb.1
+ ; RPD: 3 0
+ ;
+ ; RPD-NOPHYS-LABEL: name: physreg_liveout
+ ; RPD-NOPHYS: bb.0:
+ ; RPD-NOPHYS: SGPR VGPR
+ ;
+ ; Without physreg tracking: no live-in pressure, INLINEASM has no effect.
+ ;
+ ; RPD-NOPHYS: 0 0
+ ; RPD-NOPHYS: 2 0 %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 4 0 %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 2 0 INLINEASM &"s_mov_b32 $0, 0"{{.*}} implicit-def $sgpr10
+ ; RPD-NOPHYS: 2 0
+ ; RPD-NOPHYS: 2 0 S_BRANCH %bb.1
+ ; RPD-NOPHYS: 2 0
+ bb.0:
+ liveins: $sgpr8_sgpr9
+ successors: %bb.1
+
+ %0:sgpr_64(p4) = COPY $sgpr8_sgpr9
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ INLINEASM &"s_mov_b32 $0, 0", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def $sgpr10
+ S_BRANCH %bb.1
+
+ bb.1:
+ liveins: $sgpr10
+
+ %2:vgpr_32 = COPY $sgpr10
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %3, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
new file mode 100644
index 0000000000000..d6e3b6a734eec
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
@@ -0,0 +1,68 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
+# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
+# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
+
+# Test that physical register live-ins from MBB liveins are correctly
+# included in per-region pressure when using GCN trackers in the scheduler.
+#
+# The function has two scheduling regions separated by SCHED_BARRIER.
+# $sgpr10 is live-in to the MBB but only used in the second region.
+# With physical register tracking, $sgpr10 should contribute 1 SGPR
+# of pressure in the first region (live-through). Without physical
+# tracking, it should not appear.
+
+--- |
+ define amdgpu_kernel void @physreg_livein_across_regions() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx900" }
+...
+
+# GCN-LABEL: physreg_livein_across_regions
+#
+# Region 0 (bottom of block, processed first by scheduler):
+# $sgpr10 is used here via COPY. With phys tracking, $sgpr10 contributes
+# 1 extra SGPR compared to NOPHYS.
+# GCN: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 3
+# GCN: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 3
+#
+# Region 1 (top of block, processed second):
+# $sgpr10 is live-through (MBB live-in, not used until region 0).
+# Physical live-in tracking captures it in per-region pressure.
+# GCN: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 5
+# GCN: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 5
+
+# GCN-NOPHYS-LABEL: physreg_livein_across_regions
+#
+# Region 0: $sgpr10 not tracked, only virtual regs contribute.
+# GCN-NOPHYS: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 2
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 2
+#
+# Region 1: No physical register pressure.
+# GCN-NOPHYS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 4
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 4
+
+---
+name: physreg_livein_across_regions
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5, $sgpr10
+
+ ; Region 1: virtual reg defs only, $sgpr10 is live-through.
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ SCHED_BARRIER 0
+
+ ; Region 2: uses $sgpr10.
+ %2:vgpr_32 = COPY $sgpr10
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %3, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
new file mode 100644
index 0000000000000..0be216846f4be
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
@@ -0,0 +1,84 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
+# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
+# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
+
+# Test that physical register live-outs (derived from successor block liveins)
+# are correctly included in per-region pressure when using GCN trackers.
+#
+# bb.0 defines $sgpr10 in the first scheduling region and branches to bb.1
+# which has $sgpr10 as a live-in. This makes $sgpr10 a live-out of bb.0.
+# The second region in bb.0 does not touch $sgpr10, so it is live-through.
+# With physical register tracking, $sgpr10 should contribute 1 SGPR of
+# pressure in that second region. Without physical tracking, it should not.
+
+--- |
+ define amdgpu_kernel void @physreg_liveout_across_regions() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx900" }
+...
+
+# GCN-LABEL: physreg_liveout_across_regions
+#
+# Region 0 (bottom of bb.0, processed first by scheduler):
+# $sgpr10 is live-through here (defined above, live-out to bb.1).
+# With phys tracking it contributes +1 SGPR compared to NOPHYS.
+# GCN: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5
+# GCN: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5
+#
+# Region 1 (top of bb.0, processed second):
+# $sgpr4_sgpr5 live-in (+2) and $sgpr10 def (+1) add physical pressure.
+# GCN: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 4
+# GCN: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 4
+#
+# bb.1: $sgpr10 is a physical live-in here.
+# GCN: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 1
+# GCN: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 1
+
+# GCN-NOPHYS-LABEL: physreg_liveout_across_regions
+#
+# Region 0: $sgpr10 live-out not tracked, only virtual regs contribute.
+# GCN-NOPHYS: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 4
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 4
+#
+# Region 1: No physical register pressure.
+# GCN-NOPHYS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 2
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 2
+#
+# bb.1: $sgpr10 live-in not tracked.
+# GCN-NOPHYS: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 0
+# GCN-NOPHYS: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 0
+
+---
+name: physreg_liveout_across_regions
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ successors: %bb.1
+
+ ; Region 1: defines $sgpr10, which is live-out to bb.1.
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ $sgpr10 = S_MOV_B32 42
+ SCHED_BARRIER 0
+
+ ; Region 0: $sgpr10 is live-through (defined above, live-out to bb.1).
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %2, %2, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_BRANCH %bb.1
+
+ bb.1:
+ liveins: $sgpr10
+
+ %3:vgpr_32 = COPY $sgpr10
+ %4:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %4, %3, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index c1ca7609df9f1..b2b73e9a96fcb 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,5 +1,6 @@
-; RUN: not --crash llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs 2>&1 < %s | FileCheck -check-prefixes=ERR-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
@@ -16,13 +17,13 @@
i64 ; vcc
}
-; With the tracker enabled, no separate WWM VGPR is available and the SGPR
-; spill falls back to memory. This case cannot preserve EXEC because SCC is
-; live and no SGPR can be scavenged.
-; ERR-GCNTRACKERS: unhandled SGPR spill to memory
+; GCN-TRACKERS-NOT: ran out of registers during register allocation
; GCN-NOT: ran out of registers during register allocation
+; GCN-NOPHYS-FAIL: ran out of registers during register allocation
-; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
+; GCN Trackers now track physical register pressure correctly, so this test
+; verifies that both trackers can successfully handle code with heavy physical
+; register usage from inline assembly.
define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0 {
%alloca0 = alloca [4096 x i32], align 64, addrspace(5)
@@ -65,3 +66,4 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
attributes #0 = { nounwind alignstack=64 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
attributes #1 = { nounwind alignstack=16 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
+
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index f2bc619b2980e..91a071dd69049 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -1,17 +1,32 @@
-; RUN: llc -mtriple=amdgpu6.00-amd-amdhsa -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 < %s | FileCheck --check-prefix=GCN %s
-; RUN: llc -mtriple=amdgpu6.00-amd-amdhsa -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck --check-prefix=GCN-GCNTRACKERS %s
-
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
+; RUN: FileCheck --check-prefix=SCHED %s < %t
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-GCNTRACKERS %s
+; RUN: FileCheck --check-prefix=SCHED-GCNTRACKERS %s < %t
+; REQUIRES: asserts
; CHECK-LABEL: {{^}}spill:
; GCN: NumSgprs: 104
; GCN-GCNTRACKERS: NumSgprs: 104
; GCN: NumVgprs: 1
-; GCN-GCNTRACKERS: NumVgprs: 2
+; GCN-GCNTRACKERS: NumVgprs: 1
; GCN: ScratchSize: 0
; GCN-GCNTRACKERS: ScratchSize: 0
; GCN: Occupancy: 4
; GCN-GCNTRACKERS: Occupancy: 4
-
-; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
+;
+; Check scheduling pressure values:
+; SCHED-LABEL: spill:%bb.0 entry
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 98
+; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 97
+;
+; SCHED-GCNTRACKERS-LABEL: spill:%bb.0 entry
+; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
+; SCHED-GCNTRACKERS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
+;
+; NOTE: GCN Trackers now track pressure from both virtual and physical registers.
+; The GCN tracker now matches the generic tracker's VGPR count (1 VGPR).
+; When a live range is not found for a physical regunit, we conservatively
+; assume the unit is live, so Region SGPR pressure can be higher (193 vs 98).
+; Pressure after scheduling remains 98 vs 97 due to physical register tracking.
define amdgpu_kernel void @spill(ptr addrspace(1) %arg, i32 %cnd) #0 {
entry:
@@ -247,9 +262,15 @@ bb3:
; GCN: NumSgprs: 104
; GCN-GCNTRACKERS: NumSgprs: 104
; GCN: NumVgprs: 2
-; GCN-GCNTRACKERS: NumVgprs: 3
+; GCN-GCNTRACKERS: NumVgprs: 2
; GCN: ScratchSize: 8
-; GCN-GCNTRACKERS: ScratchSize: 12
+; GCN-GCNTRACKERS: ScratchSize: 8
+;
+; SCHED-LABEL: spill_func:%bb.0 entry
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 97
+;
+; SCHED-GCNTRACKERS-LABEL: spill_func:%bb.0 entry
+; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
define void @spill_func(ptr addrspace(1) %arg) #0 {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
new file mode 100644
index 0000000000000..7c947f4a942a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
@@ -0,0 +1,630 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
+; RUN: FileCheck --check-prefix=GCN-DEBUG %s < %t
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=NO-GCN %s
+; RUN: FileCheck --check-prefix=GENERIC-DEBUG %s < %t
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
+; RUN: FileCheck --check-prefix=GCN-NOPHYS-DEBUG %s < %t
+; REQUIRES: asserts
+
+; Test that GCN trackers correctly track physical register pressure from inline asm
+
+; GCN-DEBUG-LABEL: test_single_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_single_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_single_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_single_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_single_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_single_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_single_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 $0, 0", "={s10}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test multiple physical registers
+
+; GCN-DEBUG-LABEL: test_multiple_physregs
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_multiple_physregs
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_multiple_physregs
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_multiple_physregs(ptr addrspace(1) %out) {
+; GCN-LABEL: test_multiple_physregs:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_add_i32 s4, s10, s11
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_multiple_physregs:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_add_i32 s4, s10, s11
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s4
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_multiple_physregs:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_add_i32 s4, s10, s11
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %result = call { i32, i32 } asm sideeffect "s_mov_b32 $0, 0; s_mov_b32 $1, 1", "={s10},={s11}"()
+ %r0 = extractvalue { i32, i32 } %result, 0
+ %r1 = extractvalue { i32, i32 } %result, 1
+ %sum = add i32 %r0, %r1
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register with virtual registers
+
+; GCN-DEBUG-LABEL: test_physreg_with_vreg
+; GCN-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 10, LVGPR WT: 0, LSGPR WT: 12
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
+
+; GENERIC-DEBUG-LABEL: test_physreg_with_vreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+
+; GCN-NOPHYS-DEBUG-LABEL: test_physreg_with_vreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+
+define amdgpu_kernel void @test_physreg_with_vreg(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GCN-LABEL: test_physreg_with_vreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s4, s2
+; GCN-NEXT: s_mov_b32 s5, s3
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_physreg_with_vreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s7, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s6, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: s_mov_b32 s4, s0
+; NO-GCN-NEXT: s_mov_b32 s5, s1
+; NO-GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: s_mov_b32 s4, s2
+; NO-GCN-NEXT: s_mov_b32 s5, s3
+; NO-GCN-NEXT: s_waitcnt vmcnt(0)
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_physreg_with_vreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s6, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s0
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s1
+; GCN-NOPHYS-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s2
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s3
+; GCN-NOPHYS-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %asm_val = call i32 asm sideeffect "s_mov_b32 $0, 0", "={s10}"()
+ %val = load i32, ptr addrspace(1) %in
+ %sum = add i32 %asm_val, %val
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; Test early-clobber constraint
+
+; GCN-DEBUG-LABEL: test_early_clobber
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_early_clobber
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_early_clobber(ptr addrspace(1) %out) {
+; GCN-LABEL: test_early_clobber:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_early_clobber:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_early_clobber:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 $0, 0", "=&{s10}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test early-clobber constraint with a tuple (64-bit) register.
+; The input s12 and early-clobber output s[10:11] have distinct live ranges.
+
+; GCN-DEBUG-LABEL: test_early_clobber_tuple
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 9
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GENERIC-DEBUG-LABEL: test_early_clobber_tuple
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber_tuple
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_early_clobber_tuple(ptr addrspace(1) %out) {
+; GCN-LABEL: test_early_clobber_tuple:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s12, 42
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[10:11], s12
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_early_clobber_tuple:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s12, 42
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[10:11], s12
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_early_clobber_tuple:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s12, 42
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[10:11], s12
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %in = call i32 asm sideeffect "s_mov_b32 $0, 42", "={s12}"()
+ %val = call i64 asm sideeffect "s_mov_b64 $0, $1", "=&{s[10:11]},{s12}"(i32 %in)
+ %lo = trunc i64 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register input
+
+; GCN-DEBUG-LABEL: test_physreg_input
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GENERIC-DEBUG-LABEL: test_physreg_input
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_physreg_input
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_physreg_input(ptr addrspace(1) %out) {
+; GCN-LABEL: test_physreg_input:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_physreg_input:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s11
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_physreg_input:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 5; s_add_u32 s11, s10, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i32 asm sideeffect "s_mov_b32 s10, 5; s_add_u32 $0, s10, 1", "={s11}"()
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register pressure for tuple (64-bit) registers.
+; GCN tracker counts the 2 SGPRs.
+
+; GCN-DEBUG-LABEL: test_tuple_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GENERIC-DEBUG-LABEL: test_tuple_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+; GCN-NOPHYS-DEBUG-LABEL: test_tuple_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+
+define amdgpu_kernel void @test_tuple_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_tuple_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[10:11], 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_tuple_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[10:11], 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s10
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_tuple_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[10:11], 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i64 asm sideeffect "s_mov_b64 $0, 0", "={s[10:11]}"()
+ %lo = trunc i64 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test physical register pressure for 128-bit tuple.
+; GCN tracker counts the 4 SGPRs.
+
+; GCN-DEBUG-LABEL: test_tuple128_physreg
+; GCN-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+
+; GENERIC-DEBUG-LABEL: test_tuple128_physreg
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+; GCN-NOPHYS-DEBUG-LABEL: test_tuple128_physreg
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+
+define amdgpu_kernel void @test_tuple128_physreg(ptr addrspace(1) %out) {
+; GCN-LABEL: test_tuple128_physreg:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_tuple128_physreg:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; NO-GCN-NEXT: s_mov_b32 s3, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s2, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: v_mov_b32_e32 v0, s8
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_tuple128_physreg:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b64 s[8:11], 0; s_mov_b64 s[8:11]+2, 0
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %val = call i128 asm sideeffect "s_mov_b64 $0, 0; s_mov_b64 $0+2, 0", "={s[8:11]}"()
+ %lo = trunc i128 %val to i32
+ store i32 %lo, ptr addrspace(1) %out
+ ret void
+}
+
+; Test virtual and physical register overlap
+
+; GCN-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GCN-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 16, LVGPR WT: 0, LSGPR WT: 16
+; GCN-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
+
+; GENERIC-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+
+; GCN-NOPHYS-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
+; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
+; GCN-NOPHYS-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+
+define amdgpu_kernel void @test_vreg_and_physreg_live_range_overlap(ptr addrspace(1) %in1, ptr addrspace(1) %in2, ptr addrspace(1) %out) {
+; GCN-LABEL: test_vreg_and_physreg_live_range_overlap:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_mov_b32 s0, s2
+; GCN-NEXT: s_mov_b32 s1, s3
+; GCN-NEXT: s_mov_b32 s2, s6
+; GCN-NEXT: s_mov_b32 s3, s7
+; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s14, s6
+; GCN-NEXT: s_mov_b32 s15, s7
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; GCN-NEXT: s_endpgm
+;
+; NO-GCN-LABEL: test_vreg_and_physreg_live_range_overlap:
+; NO-GCN: ; %bb.0: ; %entry
+; NO-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; NO-GCN-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; NO-GCN-NEXT: s_mov_b32 s7, 0xf000
+; NO-GCN-NEXT: s_mov_b32 s6, -1
+; NO-GCN-NEXT: ;;#ASMSTART
+; NO-GCN-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; NO-GCN-NEXT: ;;#ASMEND
+; NO-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; NO-GCN-NEXT: s_mov_b32 s4, s0
+; NO-GCN-NEXT: s_mov_b32 s5, s1
+; NO-GCN-NEXT: s_mov_b32 s0, s2
+; NO-GCN-NEXT: s_mov_b32 s1, s3
+; NO-GCN-NEXT: s_mov_b32 s2, s6
+; NO-GCN-NEXT: s_mov_b32 s3, s7
+; NO-GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; NO-GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; NO-GCN-NEXT: s_mov_b32 s14, s6
+; NO-GCN-NEXT: s_mov_b32 s15, s7
+; NO-GCN-NEXT: s_waitcnt vmcnt(0)
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; NO-GCN-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; NO-GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; NO-GCN-NEXT: s_endpgm
+;
+; GCN-NOPHYS-LABEL: test_vreg_and_physreg_live_range_overlap:
+; GCN-NOPHYS: ; %bb.0: ; %entry
+; GCN-NOPHYS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOPHYS-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd
+; GCN-NOPHYS-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NOPHYS-NEXT: s_mov_b32 s6, -1
+; GCN-NOPHYS-NEXT: ;;#ASMSTART
+; GCN-NOPHYS-NEXT: s_mov_b32 s10, 0; s_mov_b32 s11, 1
+; GCN-NOPHYS-NEXT: ;;#ASMEND
+; GCN-NOPHYS-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOPHYS-NEXT: s_mov_b32 s4, s0
+; GCN-NOPHYS-NEXT: s_mov_b32 s5, s1
+; GCN-NOPHYS-NEXT: s_mov_b32 s0, s2
+; GCN-NOPHYS-NEXT: s_mov_b32 s1, s3
+; GCN-NOPHYS-NEXT: s_mov_b32 s2, s6
+; GCN-NOPHYS-NEXT: s_mov_b32 s3, s7
+; GCN-NOPHYS-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; GCN-NOPHYS-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; GCN-NOPHYS-NEXT: s_mov_b32 s14, s6
+; GCN-NOPHYS-NEXT: s_mov_b32 s15, s7
+; GCN-NOPHYS-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s10, v0
+; GCN-NOPHYS-NEXT: v_add_i32_e32 v0, vcc, s11, v0
+; GCN-NOPHYS-NEXT: buffer_store_dword v0, off, s[12:15], 0
+; GCN-NOPHYS-NEXT: s_endpgm
+entry:
+ %result = call { i32, i32 } asm sideeffect "s_mov_b32 $0, 0; s_mov_b32 $1, 1", "={s10},={s11}"()
+ %val1 = load i32, ptr addrspace(1) %in1
+ %val2 = load i32, ptr addrspace(1) %in2
+ %sum = add i32 %val1, %val2
+ %r0 = extractvalue { i32, i32 } %result, 0
+ %r1 = extractvalue { i32, i32 } %result, 1
+ %with_asm = add i32 %sum, %r0
+ %final = add i32 %with_asm, %r1
+ store i32 %final, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
index 616260a7fdbc7..62d97dd3db818 100644
--- a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
+++ b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
@@ -61,7 +61,8 @@ body: |
// Track pressure across MBB1.
{
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// There is a non-debug instruction in bb.1 (%2's def), so advance should
// return true.
@@ -77,7 +78,8 @@ body: |
// Track pressure just across the first debug value of bb.1.
{
MachineBasicBlock::iterator Dbg1 = std::next(MBB1.begin());
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// The following unpacks a call to
// advance(*MBB1.begin(), Dbg1, [MBB1LiveIns|nullptr])
@@ -128,7 +130,8 @@ body: |
LIS, MF.getRegInfo());
MachineBasicBlock &MBB1 = *MF.getBlockNumbered(1);
- GCNDownwardRPTracker RPTracker(LIS), RPTrackerNoLiveIns(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MF.getRegInfo()),
+ RPTrackerNoLiveIns(LIS, MF.getRegInfo());
// The following unpacks a call to
// advance(MBB1.begin(), MBB1.end(), [MBB1LiveIns|nullptr])
>From e636cdf810b62b07a59a8af2a5ee06eb8721924e Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Fri, 3 Jul 2026 20:29:37 -0500
Subject: [PATCH 2/5] De-coupled physical register tracking from GCN trackers.
Physical register tracking is enabled when GCNRPTracker is used.
It can be disabled globally with -amdgpu-track-physregs-in-gcn-trackers=false.
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 31 +-
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 21 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 14 -
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 414 +++++++--------
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 350 ++++++-------
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 478 +++++++++---------
llvm/test/CodeGen/AMDGPU/addrspacecast.ll | 12 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 346 +++++++++----
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 462 ++++++++---------
...ffer-fat-pointers-contents-legalization.ll | 74 +--
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 10 +-
...rval-bug-in-rename-independent-subregs.mir | 8 +-
llvm/test/CodeGen/AMDGPU/memory_clause.mir | 3 +-
.../AMDGPU/preserve-wwm-copy-dst-reg.ll | 8 +-
.../regpressure-physreg-early-clobber.mir | 2 +-
.../AMDGPU/regpressure-physreg-limits.mir | 4 +-
.../CodeGen/AMDGPU/regpressure_printer.mir | 156 +++---
.../CodeGen/AMDGPU/sched-physreg-liveins.mir | 2 +-
.../CodeGen/AMDGPU/sched-physreg-liveouts.mir | 2 +-
.../schedule-amdgpu-tracker-physreg-crash.ll | 2 +-
.../AMDGPU/schedule-amdgpu-tracker-physreg.ll | 17 +-
.../AMDGPU/schedule-gcn-physreg-pressure.ll | 36 +-
.../CodeGen/AMDGPU/soft-clause-dbg-value.mir | 1 -
23 files changed, 1301 insertions(+), 1152 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 0bfbb6a94b2d8..ffbb9e8f76b4c 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -22,6 +22,17 @@ using namespace llvm;
#define DEBUG_TYPE "machine-scheduler"
+static cl::opt<bool> TrackPhysRegInGCNTrackers(
+ "amdgpu-track-physregs-in-gcn-trackers", cl::Hidden,
+ cl::desc("Track physical registers (e.g. from inline asm) in the GCN "
+ "Up/Down RP trackers. Enabled by default; acts as a global "
+ "safety switch to control physical register pressure tracking."),
+ cl::init(true));
+
+bool GCNRPTracker::physRegTrackingEnabled() const {
+ return TrackPhysRegInGCNTrackers;
+}
+
bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
const GCNRPTracker::LiveRegSet &S2) {
if (S1.size() != S2.size())
@@ -642,9 +653,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
VirtLiveRegs = llvm::getVirtLiveRegs(SI, LIS, MRI);
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
- updatePhysRegTracking();
- // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
- // stale data if physical tracking was previously enabled.
+ // Always clear PhysLiveRegUnits even when physical tracking is disabled, to
+ // avoid stale data if physical tracking was previously enabled.
PhysLiveRegUnits.reset();
}
@@ -657,9 +667,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
this->VirtLiveRegs = VirtLiveRegs;
MaxPressure = CurPressure = getVirtRegPressure(MRI, VirtLiveRegs);
- updatePhysRegTracking();
- // Always clear PhysLiveRegUnits even when TrackPhysRegs is false, to avoid
- // stale data if physical tracking was previously enabled.
+ // Always clear PhysLiveRegUnits even when physical tracking is disabled, to
+ // avoid stale data if physical tracking was previously enabled.
PhysLiveRegUnits.reset();
}
@@ -671,7 +680,7 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRInfo,
}
void GCNRPTracker::initPhysLiveUnits(const BitVector &PhysLiveUnits) {
- if (!TrackPhysRegs)
+ if (!physRegTrackingEnabled())
return;
PhysLiveRegUnits = PhysLiveUnits;
GCNRegPressure PhysPressure = constructPhysRegPressure();
@@ -753,7 +762,7 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
CurPressure.inc(U.VRegOrUnit.asVirtualReg(), PrevMask, LiveMask, *MRI);
}
- if (TrackPhysRegs) {
+ if (physRegTrackingEnabled()) {
for (const MachineOperand &MO : MI.all_uses()) {
if (!MO.readsReg())
continue;
@@ -804,7 +813,7 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
else
GCNRPTracker::reset(*MI.getParent(), /*End=*/true);
- if (SeedPhysMBB && TrackPhysRegs &&
+ if (SeedPhysMBB && physRegTrackingEnabled() &&
MI.getMF()->getProperties().hasTracksLiveness())
initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveins());
@@ -1036,7 +1045,7 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
LaneBitmask NewMask = LiveMask & ~LastUseMask;
PostUseMask[Reg] = NewMask;
TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
- } else if (TrackPhysRegs) {
+ } else if (physRegTrackingEnabled()) {
MCRegUnit Unit = Use.VRegOrUnit.asMCRegUnit();
unsigned U = static_cast<unsigned>(Unit);
if (PhysLiveRegUnits.test(U) && !isUnitLiveAt(Unit, SlotIdx))
@@ -1060,7 +1069,7 @@ GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
LaneBitmask NewMask = LiveMask | Def.LaneMask;
TempPressure.inc(Reg, LiveMask, NewMask, *MRI);
- } else if (TrackPhysRegs) {
+ } else if (physRegTrackingEnabled()) {
MCRegUnit Unit = Def.VRegOrUnit.asMCRegUnit();
unsigned U = static_cast<unsigned>(Unit);
if (!PhysLiveRegUnits.test(U))
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 92819b15eaaf2..de2c0ad2f825b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -354,18 +354,12 @@ class GCNRPTracker {
GCNRegPressure CurPressure, MaxPressure;
- // Flag to control whether physical register tracking is active.
- // Set to true when GCNTrackers are enabled, false otherwise.
- bool TrackPhysRegs = false;
-
const MachineInstr *LastTrackedMI = nullptr;
GCNRPTracker(const LiveIntervals &LIS, const MachineRegisterInfo &MRI)
: LIS(LIS), MRI(&MRI),
SRI(static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo())),
- PhysLiveRegUnits(SRI->getNumRegUnits()) {
- updatePhysRegTracking();
- }
+ PhysLiveRegUnits(SRI->getNumRegUnits()) {}
/// Resets tracker before or \p After the provided \p MI, which can be a debug
/// instruction.
@@ -383,7 +377,8 @@ class GCNRPTracker {
LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;
bool shouldTrackPhysReg(Register Reg) const {
- return TrackPhysRegs && Reg.isPhysical() && MRI->isAllocatable(Reg);
+ return physRegTrackingEnabled() && Reg.isPhysical() &&
+ MRI->isAllocatable(Reg);
}
// Check if a register unit is live at a given slot index per LIS.
@@ -405,9 +400,8 @@ class GCNRPTracker {
GCNRegPressure &Pressure);
public:
- // Enable physical register tracking only if both GCNTrackers and
- // TrackPhysRegInTrackers are true.
- void updatePhysRegTracking();
+ // Returns whether physical register tracking is enabled.
+ bool physRegTrackingEnabled() const;
/// Resets tracker with the provided \p VirtLiveRegs.
void reset(const MachineRegisterInfo &MRI, const LiveRegSet &VirtLiveRegs);
@@ -425,7 +419,8 @@ class GCNRPTracker {
/// and update CurPressure/MaxPressure accordingly.
template <typename RangeT>
void initPhysLiveUnitsFromRegMaskPairs(RangeT &&Pairs) {
- assert(TrackPhysRegs && "physical register tracking must be enabled");
+ assert(physRegTrackingEnabled() &&
+ "physical register tracking must be enabled");
for (const auto &RM : Pairs)
if (MRI->isAllocatable(RM.PhysReg))
addUnitsAndIncPressure(RM.PhysReg, CurPressure);
@@ -463,7 +458,7 @@ class GCNUpwardRPTracker : public GCNRPTracker {
void reset(const MachineRegisterInfo &MRI, SlotIndex SI,
const MachineBasicBlock *SeedPhysMBB = nullptr) {
GCNRPTracker::reset(MRI, llvm::getVirtLiveRegs(SI, LIS, MRI));
- if (SeedPhysMBB && TrackPhysRegs &&
+ if (SeedPhysMBB && physRegTrackingEnabled() &&
SeedPhysMBB->getParent()->getProperties().hasTracksLiveness())
initPhysLiveUnitsFromRegMaskPairs(SeedPhysMBB->liveouts());
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index df918e7d35a4e..d880f9eeee6ca 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -79,12 +79,6 @@ static cl::opt<bool> GCNTrackers(
cl::desc("Use the AMDGPU specific RPTrackers during scheduling"),
cl::init(false));
-static cl::opt<bool> TrackPhysRegInTrackers(
- "amdgpu-trackers-physical-register-tracking", cl::Hidden,
- cl::desc("When using GCN trackers, count physical registers (e.g. from "
- "inline asm) in pressure."),
- cl::init(true));
-
static cl::opt<unsigned> PendingQueueLimit(
"amdgpu-scheduler-pending-queue-limit", cl::Hidden,
cl::desc(
@@ -215,14 +209,6 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
<< ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
}
-void GCNRPTracker::updatePhysRegTracking() {
- if (!GCNTrackers || !TrackPhysRegInTrackers) {
- TrackPhysRegs = false;
- return;
- }
- TrackPhysRegs = true;
-}
-
/// Checks whether \p SU can use the cached DAG pressure diffs to compute the
/// current register pressure.
///
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 6844b698473c7..3a54f1a1dfd87 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -8170,250 +8170,250 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6-LABEL: v_fshl_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshr_b64 v[23:24], v[8:9], 1
; GFX6-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX6-NEXT: v_not_b32_e32 v18, 63
-; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
-; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v18
-; GFX6-NEXT: v_lshr_b64 v[23:24], v[0:1], v23
-; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], v19
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v27
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 31, v10
+; GFX6-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
+; GFX6-NEXT: v_or_b32_e32 v24, v24, v8
+; GFX6-NEXT: v_mov_b32_e32 v8, 0x7f
+; GFX6-NEXT: v_lshr_b64 v[17:18], v[0:1], v17
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v19
+; GFX6-NEXT: v_lshr_b64 v[9:10], v[10:11], 1
+; GFX6-NEXT: v_bfi_b32 v11, v16, 0, v8
+; GFX6-NEXT: v_sub_i32_e32 v16, vcc, 64, v11
+; GFX6-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[9:10], v16
+; GFX6-NEXT: v_lshr_b64 v[25:26], v[23:24], v11
+; GFX6-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX6-NEXT: v_or_b32_e32 v22, v25, v16
+; GFX6-NEXT: v_or_b32_e32 v25, v26, v17
+; GFX6-NEXT: v_not_b32_e32 v26, 63
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v19, v26
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[0:1], v16
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[8:9], 1
-; GFX6-NEXT: v_mov_b32_e32 v17, 0x7f
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 31, v10
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], 1
-; GFX6-NEXT: v_bfi_b32 v10, v16, 0, v17
-; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
-; GFX6-NEXT: v_add_i32_e32 v16, vcc, v10, v18
-; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v10
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], v10
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[0:1], v10
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v21
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], v16
-; GFX6-NEXT: v_or_b32_e32 v10, v10, v21
-; GFX6-NEXT: v_or_b32_e32 v11, v11, v22
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v20
-; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[4:5], v10
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[6:7], v16
-; GFX6-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX6-NEXT: v_add_i32_e32 v19, vcc, v16, v18
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v16
-; GFX6-NEXT: v_or_b32_e32 v16, v10, v21
-; GFX6-NEXT: v_or_b32_e32 v21, v11, v22
-; GFX6-NEXT: v_lshl_b64 v[10:11], v[4:5], v19
-; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[12:13], 1
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 31, v14
-; GFX6-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[14:15], 1
-; GFX6-NEXT: v_bfi_b32 v14, v20, 0, v17
-; GFX6-NEXT: v_add_i32_e32 v18, vcc, v14, v18
-; GFX6-NEXT: v_sub_i32_e32 v16, vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v18, v17, v3, s[4:5]
+; GFX6-NEXT: v_add_i32_e64 v2, s[4:5], v11, v26
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[9:10], v2
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v19
+; GFX6-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v11
+; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v25, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[9:10], v11
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX6-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX6-NEXT: v_and_b32_e32 v19, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, v0, s[4:5]
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 64, v19
+; GFX6-NEXT: v_or_b32_e32 v0, v17, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v16, v9
+; GFX6-NEXT: v_lshr_b64 v[9:10], v[4:5], v3
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[6:7], v19
+; GFX6-NEXT: v_or_b32_e32 v3, v18, v11
+; GFX6-NEXT: v_or_b32_e32 v11, v9, v16
+; GFX6-NEXT: v_or_b32_e32 v16, v10, v17
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v19, v26
+; GFX6-NEXT: v_lshl_b64 v[9:10], v[4:5], v19
+; GFX6-NEXT: v_lshl_b64 v[4:5], v[4:5], v17
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v18, 0, v10, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v19
+; GFX6-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v19, v5, v7, vcc
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[12:13], 1
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 31, v14
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], 1
+; GFX6-NEXT: v_bfi_b32 v14, v20, 0, v8
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v14
+; GFX6-NEXT: v_add_i32_e32 v15, vcc, v14, v26
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[4:5], v14
+; GFX6-NEXT: v_lshl_b64 v[10:11], v[6:7], v10
+; GFX6-NEXT: v_lshr_b64 v[12:13], v[6:7], v14
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[6:7], v15
+; GFX6-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX6-NEXT: v_or_b32_e32 v9, v9, v11
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GFX6-NEXT: v_lshr_b64 v[12:13], v[10:11], v14
-; GFX6-NEXT: v_lshr_b64 v[14:15], v[8:9], v14
-; GFX6-NEXT: v_lshl_b64 v[16:17], v[10:11], v16
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[10:11], v18
-; GFX6-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX6-NEXT: v_or_b32_e32 v15, v15, v17
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v24, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v25, v1
-; GFX6-NEXT: v_or_b32_e32 v3, v23, v3
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v8
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v9
-; GFX6-NEXT: v_or_b32_e32 v6, v6, v10
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v11
+; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v12, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc
+; GFX6-NEXT: v_or_b32_e32 v4, v17, v4
+; GFX6-NEXT: v_or_b32_e32 v5, v18, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v16, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v19, v7
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshrrev_b64 v[23:24], 1, v[8:9]
; GFX8-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX8-NEXT: v_not_b32_e32 v18, 63
-; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
-; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v18
-; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v19, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v27, v[0:1]
+; GFX8-NEXT: v_lshlrev_b32_e32 v8, 31, v10
+; GFX8-NEXT: v_sub_u32_e32 v17, vcc, 64, v19
+; GFX8-NEXT: v_or_b32_e32 v24, v24, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, 0x7f
+; GFX8-NEXT: v_lshrrev_b64 v[17:18], v17, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v19, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[9:10], 1, v[10:11]
+; GFX8-NEXT: v_bfi_b32 v11, v16, 0, v8
+; GFX8-NEXT: v_sub_u32_e32 v16, vcc, 64, v11
+; GFX8-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[9:10]
+; GFX8-NEXT: v_lshrrev_b64 v[25:26], v11, v[23:24]
+; GFX8-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX8-NEXT: v_or_b32_e32 v22, v25, v16
+; GFX8-NEXT: v_or_b32_e32 v25, v26, v17
+; GFX8-NEXT: v_not_b32_e32 v26, 63
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v19, v26
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v17, 0x7f
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 31, v10
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[10:11]
-; GFX8-NEXT: v_bfi_b32 v10, v16, 0, v17
-; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
-; GFX8-NEXT: v_add_u32_e32 v16, vcc, v10, v18
-; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v10
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], v10, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v21
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v22
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v20
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[4:5]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v16, v[6:7]
-; GFX8-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX8-NEXT: v_add_u32_e32 v19, vcc, v16, v18
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v16, v[4:5]
-; GFX8-NEXT: v_or_b32_e32 v16, v10, v21
-; GFX8-NEXT: v_or_b32_e32 v21, v11, v22
-; GFX8-NEXT: v_lshlrev_b64 v[10:11], v19, v[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
-; GFX8-NEXT: v_lshlrev_b32_e32 v10, 31, v14
-; GFX8-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], 1, v[14:15]
-; GFX8-NEXT: v_bfi_b32 v14, v20, 0, v17
-; GFX8-NEXT: v_add_u32_e32 v18, vcc, v14, v18
-; GFX8-NEXT: v_sub_u32_e32 v16, vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v18, v17, v3, s[4:5]
+; GFX8-NEXT: v_add_u32_e64 v2, s[4:5], v11, v26
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v2, v[9:10]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX8-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v25, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v11, v[9:10]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX8-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX8-NEXT: v_and_b32_e32 v19, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, v0, s[4:5]
+; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 64, v19
+; GFX8-NEXT: v_or_b32_e32 v0, v17, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v16, v9
+; GFX8-NEXT: v_lshrrev_b64 v[9:10], v3, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v19, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v3, v18, v11
+; GFX8-NEXT: v_or_b32_e32 v11, v9, v16
+; GFX8-NEXT: v_or_b32_e32 v16, v10, v17
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v19, v26
+; GFX8-NEXT: v_lshlrev_b64 v[9:10], v19, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v17, v[4:5]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v18, 0, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v19
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v5, v7, vcc
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], 1, v[12:13]
+; GFX8-NEXT: v_lshlrev_b32_e32 v6, 31, v14
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
+; GFX8-NEXT: v_bfi_b32 v14, v20, 0, v8
+; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v14
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v14, v26
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[12:13], v14, v[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v15, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v11
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GFX8-NEXT: v_lshrrev_b64 v[12:13], v14, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[14:15], v14, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v18, v[10:11]
-; GFX8-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX8-NEXT: v_or_b32_e32 v15, v15, v17
-; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v24, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v25, v1
-; GFX8-NEXT: v_or_b32_e32 v3, v23, v3
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v8
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v9
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v10
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, 0, v12, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc
+; GFX8-NEXT: v_or_b32_e32 v4, v17, v4
+; GFX8-NEXT: v_or_b32_e32 v5, v18, v5
+; GFX8-NEXT: v_or_b32_e32 v6, v16, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v19, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v23, 0x7f, v16
+; GFX9-NEXT: v_and_b32_e32 v19, 0x7f, v16
; GFX9-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX9-NEXT: v_sub_u32_e32 v17, 64, v23
-; GFX9-NEXT: v_mov_b32_e32 v24, 0x7f
+; GFX9-NEXT: v_sub_u32_e32 v17, 64, v19
+; GFX9-NEXT: v_mov_b32_e32 v25, 0x7f
; GFX9-NEXT: v_lshrrev_b64 v[17:18], v17, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v19, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v9, v10, 31, v9
; GFX9-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; GFX9-NEXT: v_bfi_b32 v25, v16, 0, v24
-; GFX9-NEXT: v_sub_u32_e32 v16, 64, v25
+; GFX9-NEXT: v_bfi_b32 v26, v16, 0, v25
+; GFX9-NEXT: v_sub_u32_e32 v16, 64, v26
; GFX9-NEXT: v_or_b32_e32 v21, v17, v21
-; GFX9-NEXT: v_or_b32_e32 v22, v18, v22
; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
-; GFX9-NEXT: v_lshrrev_b64 v[18:19], v25, v[8:9]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GFX9-NEXT: v_or_b32_e32 v18, v18, v16
-; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v23
-; GFX9-NEXT: v_or_b32_e32 v19, v19, v17
+; GFX9-NEXT: v_lshrrev_b64 v[23:24], v26, v[8:9]
+; GFX9-NEXT: v_or_b32_e32 v18, v18, v22
+; GFX9-NEXT: v_or_b32_e32 v22, v23, v16
+; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v19
+; GFX9-NEXT: v_or_b32_e32 v23, v24, v17
; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v23, v[0:1]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GFX9-NEXT: v_cndmask_b32_e32 v26, 0, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v21, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v22, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v17, v0, v2, s[4:5]
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v3, s[4:5]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v0, v[10:11]
-; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v1, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v25, v[10:11]
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v25
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v17, v18, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffc0, v26
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v3, s[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v2, v[10:11]
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v19, 0, v1, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v26, v[10:11]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v22, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v23, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v0, v26, v2
-; GFX9-NEXT: v_or_b32_e32 v2, v17, v8
-; GFX9-NEXT: v_and_b32_e32 v17, 0x7f, v20
-; GFX9-NEXT: v_cndmask_b32_e64 v19, 0, v1, s[4:5]
-; GFX9-NEXT: v_or_b32_e32 v1, v18, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, 64, v17
+; GFX9-NEXT: v_or_b32_e32 v0, v18, v2
+; GFX9-NEXT: v_or_b32_e32 v2, v16, v8
+; GFX9-NEXT: v_and_b32_e32 v16, 0x7f, v20
+; GFX9-NEXT: v_cndmask_b32_e64 v21, 0, v1, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v1, v19, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, 64, v16
; GFX9-NEXT: v_lshrrev_b64 v[8:9], v3, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[10:11], v17, v[6:7]
-; GFX9-NEXT: v_or_b32_e32 v3, v16, v19
-; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v17
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX9-NEXT: v_or_b32_e32 v3, v17, v21
+; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
; GFX9-NEXT: v_or_b32_e32 v10, v8, v10
; GFX9-NEXT: v_or_b32_e32 v11, v9, v11
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], v17, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], v16, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v17, v[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v11, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v4, v6, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v4, v6, vcc
; GFX9-NEXT: v_lshrrev_b64 v[4:5], 1, v[12:13]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v8, v7, vcc
; GFX9-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
-; GFX9-NEXT: v_bfi_b32 v13, v20, 0, v24
+; GFX9-NEXT: v_bfi_b32 v13, v20, 0, v25
; GFX9-NEXT: v_lshl_or_b32 v5, v14, 31, v5
; GFX9-NEXT: v_sub_u32_e32 v10, 64, v13
; GFX9-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
@@ -8431,9 +8431,9 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v4, v16, v4
+; GFX9-NEXT: v_or_b32_e32 v4, v17, v4
; GFX9-NEXT: v_or_b32_e32 v5, v18, v5
-; GFX9-NEXT: v_or_b32_e32 v6, v17, v6
+; GFX9-NEXT: v_or_b32_e32 v6, v16, v6
; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 11b62d825c965..0725b56cf89ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -7881,182 +7881,182 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GFX6-NEXT: v_mov_b32_e32 v18, 0x7f
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], 1
+; GFX6-NEXT: v_mov_b32_e32 v19, 0x7f
+; GFX6-NEXT: v_lshl_b64 v[17:18], v[0:1], 1
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX6-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX6-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX6-NEXT: v_not_b32_e32 v17, 63
-; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
-; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v17
-; GFX6-NEXT: v_lshr_b64 v[23:24], v[21:22], v23
-; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[21:22], v19
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[21:22], v27
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7f, v16
-; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
-; GFX6-NEXT: v_add_i32_e32 v16, vcc, v2, v17
-; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v2
-; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[10:11], v2
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[8:9], v2
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[10:11], v21
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, 64, v27
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[17:18], v0
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v27
+; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX6-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, 64, v16
+; GFX6-NEXT: v_lshl_b64 v[23:24], v[10:11], v0
+; GFX6-NEXT: v_lshr_b64 v[25:26], v[8:9], v16
+; GFX6-NEXT: v_or_b32_e32 v22, v1, v22
+; GFX6-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX6-NEXT: v_not_b32_e32 v25, 63
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v27, v25
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[17:18], v0
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX6-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
+; GFX6-NEXT: v_add_i32_e64 v0, s[4:5], v16, v25
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[10:11], v0
+; GFX6-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX6-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v23, s[4:5]
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v16
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v1, v24, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[6:7]
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[17:18], v27
+; GFX6-NEXT: v_cndmask_b32_e64 v9, v2, v9, s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX6-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
-; GFX6-NEXT: v_or_b32_e32 v21, v2, v21
-; GFX6-NEXT: v_or_b32_e32 v22, v3, v22
; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], v16
-; GFX6-NEXT: v_bfi_b32 v16, v20, 0, v18
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v24, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v25, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v19, v8
-; GFX6-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v9
; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], 1
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX6-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX6-NEXT: v_or_b32_e32 v6, v6, v4
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
-; GFX6-NEXT: v_add_i32_e32 v21, vcc, v16, v17
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[8:9], v10
-; GFX6-NEXT: v_lshl_b64 v[18:19], v[6:7], v16
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 64, v16
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[8:9], v4
+; GFX6-NEXT: v_lshl_b64 v[10:11], v[6:7], v16
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v16, v25
+; GFX6-NEXT: v_or_b32_e32 v10, v4, v10
+; GFX6-NEXT: v_or_b32_e32 v11, v5, v11
; GFX6-NEXT: v_lshl_b64 v[4:5], v[8:9], v16
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[8:9], v21
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[8:9], v17
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX6-NEXT: v_or_b32_e32 v10, v10, v18
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX6-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
-; GFX6-NEXT: v_or_b32_e32 v11, v11, v19
-; GFX6-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
-; GFX6-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v17, 0, v4, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
-; GFX6-NEXT: v_add_i32_e32 v17, vcc, v6, v17
-; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v6
-; GFX6-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[14:15], v6
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[12:13], v6
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[14:15], v8
-; GFX6-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX6-NEXT: v_or_b32_e32 v9, v7, v9
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v17
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX6-NEXT: v_and_b32_e32 v10, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v8, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v9, v5, v7, vcc
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, 64, v10
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[12:13], v10
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[14:15], v6
+; GFX6-NEXT: v_add_i32_e32 v11, vcc, v10, v25
+; GFX6-NEXT: v_or_b32_e32 v16, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v19, v5, v7
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v11
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[14:15], v10
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v16, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
; GFX6-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
; GFX6-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
-; GFX6-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX6-NEXT: v_or_b32_e32 v2, v21, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v22, v3
+; GFX6-NEXT: v_or_b32_e32 v4, v17, v6
; GFX6-NEXT: v_or_b32_e32 v5, v18, v7
-; GFX6-NEXT: v_or_b32_e32 v6, v10, v8
-; GFX6-NEXT: v_or_b32_e32 v7, v11, v9
+; GFX6-NEXT: v_or_b32_e32 v6, v8, v10
+; GFX6-NEXT: v_or_b32_e32 v7, v9, v11
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v18, 0x7f
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], 1, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v19, 0x7f
+; GFX8-NEXT: v_lshlrev_b64 v[17:18], 1, v[0:1]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX8-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX8-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX8-NEXT: v_not_b32_e32 v17, 63
-; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
-; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v17
-; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[21:22]
-; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v19, v[21:22]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v27, v[21:22]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
-; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
-; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
-; GFX8-NEXT: v_and_b32_e32 v2, 0x7f, v16
-; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
-; GFX8-NEXT: v_add_u32_e32 v16, vcc, v2, v17
-; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], v2, v[10:11]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v2, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[10:11]
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 64, v27
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, v[17:18]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v27, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX8-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 64, v16
+; GFX8-NEXT: v_lshlrev_b64 v[23:24], v0, v[10:11]
+; GFX8-NEXT: v_lshrrev_b64 v[25:26], v16, v[8:9]
+; GFX8-NEXT: v_or_b32_e32 v22, v1, v22
+; GFX8-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX8-NEXT: v_not_b32_e32 v25, 63
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v27, v25
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v0, v[17:18]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX8-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
+; GFX8-NEXT: v_add_u32_e64 v0, s[4:5], v16, v25
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11]
+; GFX8-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX8-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v23, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v16
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v1, v24, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[6:7]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v27, v[17:18]
+; GFX8-NEXT: v_cndmask_b32_e64 v9, v2, v9, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; GFX8-NEXT: v_or_b32_e32 v21, v2, v21
-; GFX8-NEXT: v_or_b32_e32 v22, v3, v22
; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[10:11]
-; GFX8-NEXT: v_bfi_b32 v16, v20, 0, v18
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v24, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v25, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v19, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
; GFX8-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX8-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX8-NEXT: v_or_b32_e32 v6, v6, v4
-; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
-; GFX8-NEXT: v_add_u32_e32 v21, vcc, v16, v17
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[18:19], v16, v[6:7]
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 64, v16
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v4, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v16, v25
+; GFX8-NEXT: v_or_b32_e32 v10, v4, v10
+; GFX8-NEXT: v_or_b32_e32 v11, v5, v11
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v16, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v21, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v17, v[8:9]
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v18
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GFX8-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v19
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
-; GFX8-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v4, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
-; GFX8-NEXT: v_add_u32_e32 v17, vcc, v6, v17
-; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], v6, v[14:15]
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v6, v[12:13]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v8, v[14:15]
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX8-NEXT: v_or_b32_e32 v9, v7, v9
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v17, v[14:15]
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX8-NEXT: v_and_b32_e32 v10, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v5, v7, vcc
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 64, v10
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v10, v[12:13]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], v6, v[14:15]
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v10, v25
+; GFX8-NEXT: v_or_b32_e32 v16, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v19, v5, v7
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v11, v[14:15]
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v10, v[14:15]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v16, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
-; GFX8-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX8-NEXT: v_or_b32_e32 v2, v21, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v22, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v17, v6
; GFX8-NEXT: v_or_b32_e32 v5, v18, v7
-; GFX8-NEXT: v_or_b32_e32 v6, v10, v8
-; GFX8-NEXT: v_or_b32_e32 v7, v11, v9
+; GFX8-NEXT: v_or_b32_e32 v6, v8, v10
+; GFX8-NEXT: v_or_b32_e32 v7, v9, v11
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshr_v2i128:
@@ -8066,51 +8066,51 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_mov_b32_e32 v19, 0x7f
; GFX9-NEXT: v_lshlrev_b64 v[17:18], 1, v[0:1]
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GFX9-NEXT: v_bfi_b32 v23, v16, 0, v19
+; GFX9-NEXT: v_bfi_b32 v27, v16, 0, v19
; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX9-NEXT: v_sub_u32_e32 v0, 64, v23
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v27
; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[17:18]
-; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
-; GFX9-NEXT: v_and_b32_e32 v26, 0x7f, v16
-; GFX9-NEXT: v_or_b32_e32 v24, v0, v21
-; GFX9-NEXT: v_sub_u32_e32 v0, 64, v26
-; GFX9-NEXT: v_or_b32_e32 v25, v1, v22
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[10:11]
-; GFX9-NEXT: v_lshrrev_b64 v[21:22], v26, v[8:9]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GFX9-NEXT: v_or_b32_e32 v21, v21, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v23
-; GFX9-NEXT: v_or_b32_e32 v22, v22, v1
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v27, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v16, 0x7f, v16
+; GFX9-NEXT: v_or_b32_e32 v21, v0, v21
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v16
+; GFX9-NEXT: v_lshlrev_b64 v[23:24], v0, v[10:11]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v27
+; GFX9-NEXT: v_or_b32_e32 v22, v1, v22
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[17:18]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v24, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v25, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v0, v2, s[4:5]
-; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v26
-; GFX9-NEXT: v_lshlrev_b64 v[16:17], v23, v[17:18]
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v1, v3, s[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v21, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v27
+; GFX9-NEXT: v_lshrrev_b64 v[25:26], v16, v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v22, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v21, v0, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v16
+; GFX9-NEXT: v_cndmask_b32_e64 v22, v1, v3, s[4:5]
; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11]
-; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v26
-; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v18, v0, v21, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v21, v1, v22, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v26, v[10:11]
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v18, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v21, v9, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], v27, v[17:18]
+; GFX9-NEXT: v_or_b32_e32 v23, v25, v23
+; GFX9-NEXT: v_or_b32_e32 v24, v26, v24
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
+; GFX9-NEXT: v_cndmask_b32_e64 v23, v0, v23, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v17, v1, v24, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v16, v[10:11]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v23, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
-; GFX9-NEXT: v_or_b32_e32 v0, v16, v8
-; GFX9-NEXT: v_or_b32_e32 v1, v17, v9
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v8
+; GFX9-NEXT: v_or_b32_e32 v1, v3, v9
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v5
; GFX9-NEXT: v_bfi_b32 v16, v20, 0, v19
; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
; GFX9-NEXT: v_sub_u32_e32 v4, 64, v16
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX9-NEXT: v_or_b32_e32 v2, v21, v10
+; GFX9-NEXT: v_or_b32_e32 v3, v22, v11
; GFX9-NEXT: v_lshrrev_b64 v[4:5], v4, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index a739ede299c06..510b021a26320 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -1920,268 +1920,268 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_srem_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v10, 0x1000
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[10:11], v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v5, v7, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v4, v7
-; GISEL-NEXT: v_xor_b32_e32 v7, v8, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v5
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
-; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v5
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v7, vcc
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v8
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v4
-; GISEL-NEXT: v_trunc_f32_e32 v8, v8
-; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v8
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v4, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v4, v8
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
-; GISEL-NEXT: v_mul_lo_u32 v13, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v7, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GISEL-NEXT: v_lshl_b64 v[9:10], v[7:8], v4
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[7:8], v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v11, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v7, v1, v6
+; GISEL-NEXT: v_xor_b32_e32 v0, v9, v11
+; GISEL-NEXT: v_xor_b32_e32 v1, v10, v11
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v0
+; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v1
+; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v0
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mac_f32_e32 v9, 0x4f800000, v10
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v9, v9
+; GISEL-NEXT: v_mul_f32_e32 v9, 0x5f7ffffc, v9
+; GISEL-NEXT: v_mul_f32_e32 v10, 0x2f800000, v9
+; GISEL-NEXT: v_trunc_f32_e32 v10, v10
+; GISEL-NEXT: v_mac_f32_e32 v9, 0xcf800000, v10
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v10
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v16, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v18, v15, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v16, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
-; GISEL-NEXT: v_mul_lo_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; GISEL-NEXT: v_mul_hi_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v9
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v19, 0
-; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v19, v8
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v19, v[12:13]
-; GISEL-NEXT: v_xor_b32_e32 v15, v0, v4
-; GISEL-NEXT: v_mul_lo_u32 v0, v16, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v14
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], v6
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v9
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v16, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v18, v15, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v16, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v16, v10, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v8, v9
+; GISEL-NEXT: v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT: v_mul_lo_u32 v13, v7, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v16, v8, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v15, v8
-; GISEL-NEXT: v_xor_b32_e32 v14, v1, v4
-; GISEL-NEXT: v_mul_lo_u32 v1, v14, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v8, v10
+; GISEL-NEXT: v_mul_lo_u32 v13, v7, v10
+; GISEL-NEXT: v_mul_hi_u32 v10, v7, v10
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v8
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_mul_lo_u32 v12, v14, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v0, v1
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v8
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v16, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v8, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[1:2]
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v7, v16, v[8:9]
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v15, v0
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v15, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v15, v7
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v13, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v1, v6, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v0, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v11
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v1
-; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], v11, v1, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
-; GISEL-NEXT: v_xor_b32_e32 v8, v8, v1
-; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v8
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v16, v7
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v9
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v10, v9
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v0, v15, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v0, v13, v[10:11]
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v1, v15, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v7, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v1
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v5
-; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v7
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v0, v7, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v9, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v9
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v9
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v20, v[1:2]
-; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
-; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v5
-; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v7, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v14, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[9:10]
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc
-; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_mul_lo_u32 v9, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v23, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT: v_sub_i32_e64 v11, s[4:5], v8, v0
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v1, vcc
+; GISEL-NEXT: v_subbrev_u32_e64 v12, vcc, 0, v7, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v12, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v12, v1
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v14, vcc
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], v7, v1, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT: v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v4, v7
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v5, v7, vcc
+; GISEL-NEXT: v_xor_b32_e32 v5, v2, v6
+; GISEL-NEXT: v_xor_b32_e32 v4, v3, v6
+; GISEL-NEXT: v_xor_b32_e32 v3, v8, v7
+; GISEL-NEXT: v_xor_b32_e32 v2, v9, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v2
+; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v3
+; GISEL-NEXT: v_subb_u32_e32 v16, vcc, 0, v2, vcc
+; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v8
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v7, v7
+; GISEL-NEXT: v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v7
+; GISEL-NEXT: v_trunc_f32_e32 v8, v8
+; GISEL-NEXT: v_mac_f32_e32 v7, 0xcf800000, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v8
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v14, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v13, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v20, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v13, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v15, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v16, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v7, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v19, v14, v[9:10]
-; GISEL-NEXT: v_mul_hi_u32 v9, v14, v0
-; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
-; GISEL-NEXT: v_mul_lo_u32 v2, v16, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v13, v11
; GISEL-NEXT: v_mul_lo_u32 v10, v14, v11
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
-; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; GISEL-NEXT: v_mul_hi_u32 v9, v14, v11
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_mul_lo_u32 v10, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v7
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v8, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v14, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v13, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v13, v11
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v16, v2, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v10, v12, v2
-; GISEL-NEXT: v_xor_b32_e32 v11, v3, v7
-; GISEL-NEXT: v_mul_lo_u32 v3, v11, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v11, v0
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v13, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v14, v8, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v5, v7
+; GISEL-NEXT: v_mul_lo_u32 v10, v5, v8
+; GISEL-NEXT: v_mul_lo_u32 v11, v4, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v4, v7
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v12, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v10, v3
-; GISEL-NEXT: v_mul_lo_u32 v10, v11, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_mul_hi_u32 v10, v5, v8
+; GISEL-NEXT: v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v4, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v10, v11, v2
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v13, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v11, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v11, v3
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v8, v7
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v11, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v2, v13, v[9:10]
+; GISEL-NEXT: v_subb_u32_e64 v7, s[4:5], v4, v11, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v4, v11
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v3
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v9, vcc, v2, v6
-; GISEL-NEXT: v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v6
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v10, v8
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[4:5]
+; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v5, v3
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v2, vcc
+; GISEL-NEXT: v_subbrev_u32_e64 v10, vcc, 0, v4, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v10, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v9, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v10, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v12, vcc
+; GISEL-NEXT: v_sub_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT: v_subb_u32_e64 v2, s[4:5], v4, v2, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v6
+; GISEL-NEXT: v_xor_b32_e32 v4, v2, v6
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v3, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v4, v6, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64_pow2_shl_denom:
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 18ed68396fa4f..9a987ac54cd7a 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -1554,21 +1554,21 @@ define <16 x ptr> @addrspacecast_v16p6_to_v16p0(<16 x ptr addrspace(6)> %ptr) {
; HSA-NEXT: v_mov_b32_e32 v24, v12
; HSA-NEXT: v_mov_b32_e32 v20, v10
; HSA-NEXT: v_mov_b32_e32 v16, v8
-; HSA-NEXT: v_mov_b32_e32 v14, v7
; HSA-NEXT: v_mov_b32_e32 v12, v6
-; HSA-NEXT: v_mov_b32_e32 v10, v5
; HSA-NEXT: v_mov_b32_e32 v8, v4
-; HSA-NEXT: v_mov_b32_e32 v6, v3
; HSA-NEXT: v_mov_b32_e32 v4, v2
; HSA-NEXT: v_mov_b32_e32 v2, v1
; HSA-NEXT: v_mov_b32_e32 v1, 0
-; HSA-NEXT: v_mov_b32_e32 v3, 0
-; HSA-NEXT: v_mov_b32_e32 v5, 0
-; HSA-NEXT: v_mov_b32_e32 v7, 0
+; HSA-NEXT: v_mov_b32_e32 v6, v3
+; HSA-NEXT: v_mov_b32_e32 v10, v5
+; HSA-NEXT: v_mov_b32_e32 v14, v7
; HSA-NEXT: v_mov_b32_e32 v18, v9
; HSA-NEXT: v_mov_b32_e32 v22, v11
; HSA-NEXT: v_mov_b32_e32 v26, v13
; HSA-NEXT: v_mov_b32_e32 v30, v15
+; HSA-NEXT: v_mov_b32_e32 v3, 0
+; HSA-NEXT: v_mov_b32_e32 v5, 0
+; HSA-NEXT: v_mov_b32_e32 v7, 0
; HSA-NEXT: v_mov_b32_e32 v9, 0
; HSA-NEXT: v_mov_b32_e32 v11, 0
; HSA-NEXT: v_mov_b32_e32 v13, 0
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 41c3e0a07f5f5..d6b7d1eb83187 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -370,76 +370,151 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX908-LABEL: v32_asm_def_use:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v33, v1
-; GFX908-NEXT: v_mov_b32_e32 v34, v0
+; GFX908-NEXT: v_mov_b32_e32 v32, v1
+; GFX908-NEXT: v_mov_b32_e32 v33, v0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def v[0:31] a[0:15]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a15
-; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def v32
-; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a15
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a14
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a13
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a12
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a11
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a14
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a13
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a12
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a11
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a10
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a9
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a8
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a7
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX908-NEXT: s_nop 9
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a6
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a5
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a4
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a3
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v35
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX908-NEXT: ;;#ASMSTART
+; GFX908-NEXT: ; def v32
+; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
-; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_write_b32 a32, v35
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: s_nop 7
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a3 v[0:31]
; GFX908-NEXT: ;;#ASMEND
@@ -451,9 +526,8 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-LABEL: v32_asm_def_use:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a32 ; Reload Reuse
-; GFX90A-NEXT: v_mov_b32_e32 v34, v0
-; GFX90A-NEXT: v_mov_b32_e32 v33, v1
+; GFX90A-NEXT: v_mov_b32_e32 v33, v0
+; GFX90A-NEXT: v_mov_b32_e32 v32, v1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:31] a[0:15]
; GFX90A-NEXT: ;;#ASMEND
@@ -473,20 +547,53 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-NEXT: v_accvgpr_mov_b32 a18, a2
; GFX90A-NEXT: v_accvgpr_mov_b32 a17, a1
; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a0
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v32
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
+; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a1
+; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v35 ; Reload Reuse
-; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
@@ -1095,55 +1202,118 @@ define void @no_free_vgprs_at_sgpr_to_agpr_copy(float %v0, float %v1) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2
; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1
; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0
-; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX90A-NEXT: buffer_store_dword a10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX90A-NEXT: v_accvgpr_read_b32 v39, a11 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v38, a12 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v37, a13 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v36, a14 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a15 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX90A-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX90A-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX90A-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX90A-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX90A-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX90A-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX90A-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX90A-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX90A-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX90A-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX90A-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX90A-NEXT: v_accvgpr_read_b32 v15, a15
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX90A-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX90A-NEXT: v_accvgpr_write_b32 a11, v39 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a12, v38 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a13, v37 ; Reload Reuse
-; GFX90A-NEXT: v_accvgpr_write_b32 a14, v36 ; Reload Reuse
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a15, v35 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
+; GFX90A-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX90A-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX90A-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX90A-NEXT: v_accvgpr_write_b32 a5, v5
+; GFX90A-NEXT: v_accvgpr_write_b32 a6, v6
+; GFX90A-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX90A-NEXT: v_accvgpr_write_b32 a8, v8
+; GFX90A-NEXT: v_accvgpr_write_b32 a9, v9
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v10
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v12
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v13
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v14
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v15
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
+; GFX90A-NEXT: v_accvgpr_write_b32 a32, v34 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v34 ; Reload Reuse
; GFX90A-NEXT: s_setpc_b64 s[30:31]
%asm = call { <32 x i32>, <16 x float> } asm sideeffect "; def $0 $1","=${v[0:31]},=${s[0:15]}"()
%vgpr0 = extractvalue { <32 x i32>, <16 x float> } %asm, 0
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2b1650e429b86..6f84ac1923848 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -12,10 +12,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -62,10 +62,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -112,10 +112,10 @@ define <32 x float> @bitcast_v32i32_to_v32f32(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -636,10 +636,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -686,10 +686,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -736,10 +736,10 @@ define <32 x i32> @bitcast_v32f32_to_v32i32(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1468,10 +1468,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1518,10 +1518,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -1568,10 +1568,10 @@ define <16 x i64> @bitcast_v32i32_to_v16i64(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2092,10 +2092,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2142,10 +2142,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2192,10 +2192,10 @@ define <32 x i32> @bitcast_v16i64_to_v32i32(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2724,10 +2724,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i32_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2774,10 +2774,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -2824,10 +2824,10 @@ define <16 x double> @bitcast_v32i32_to_v16f64(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -3348,16 +3348,15 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v32i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB10_2
; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3376,22 +3375,20 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; SI-NEXT: .LBB10_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v32i32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; VI-NEXT: s_cbranch_execz .LBB10_2
; VI-NEXT: ; %bb.1: ; %cmp.true
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3410,22 +3407,20 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; VI-NEXT: .LBB10_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB10_2
; GFX9-NEXT: ; %bb.1: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -3444,7 +3439,6 @@ define <32 x i32> @bitcast_v16f64_to_v32i32(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; GFX9-NEXT: .LBB10_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v32i32:
@@ -22330,10 +22324,10 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -22380,10 +22374,10 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -30332,10 +30326,10 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -30382,10 +30376,10 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -31988,10 +31982,10 @@ define <32 x i32> @bitcast_v64f16_to_v32i32(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -33548,10 +33542,10 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32i32_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -33598,10 +33592,10 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32i32_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -35035,10 +35029,10 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v32i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36157,10 +36151,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36207,10 +36201,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36257,10 +36251,10 @@ define <16 x i64> @bitcast_v32f32_to_v16i64(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -36989,10 +36983,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37039,10 +37033,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37089,10 +37083,10 @@ define <32 x float> @bitcast_v16i64_to_v32f32(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37621,10 +37615,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f32_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37671,10 +37665,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -37721,10 +37715,10 @@ define <16 x double> @bitcast_v32f32_to_v16f64(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -38453,16 +38447,15 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB34_2
; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38481,22 +38474,20 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; SI-NEXT: .LBB34_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v32f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; VI-NEXT: s_cbranch_execz .LBB34_2
; VI-NEXT: ; %bb.1: ; %cmp.true
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38515,22 +38506,20 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; VI-NEXT: .LBB34_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB34_2
; GFX9-NEXT: ; %bb.1: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -38549,7 +38538,6 @@ define <32 x float> @bitcast_v16f64_to_v32f32(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
; GFX9-NEXT: .LBB34_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v32f32:
@@ -58340,10 +58328,10 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -58390,10 +58378,10 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -66540,10 +66528,10 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -66590,10 +66578,10 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -68343,10 +68331,10 @@ define <32 x float> @bitcast_v64f16_to_v32f32(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -69903,10 +69891,10 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v32f32_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -69953,10 +69941,10 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v32f32_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -71537,10 +71525,10 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v32f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72659,10 +72647,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i64_to_v16f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72709,10 +72697,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v16f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -72759,10 +72747,10 @@ define <16 x double> @bitcast_v16i64_to_v16f64(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73290,10 +73278,10 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f64_to_v16i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73314,20 +73302,18 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; SI-NEXT: .LBB54_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v16i64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73348,20 +73334,18 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: .LBB54_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -73382,11 +73366,9 @@ define <16 x i64> @bitcast_v16f64_to_v16i64(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: .LBB54_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v16i64:
@@ -92355,10 +92337,10 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -92405,10 +92387,10 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -100338,10 +100320,10 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -100388,10 +100370,10 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -102002,10 +101984,10 @@ define <16 x i64> @bitcast_v64f16_to_v16i64(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -103562,10 +103544,10 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16i64_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -103612,10 +103594,10 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v16i64_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -105057,10 +105039,10 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v16i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -125347,10 +125329,10 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -125364,7 +125346,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -125375,16 +125356,15 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB76_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -125398,7 +125378,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -125409,7 +125388,6 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB76_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64bf16:
@@ -133435,10 +133413,10 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -133452,7 +133430,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -133463,16 +133440,15 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB80_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -133486,7 +133462,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -133497,7 +133472,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB80_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64f16:
@@ -135143,10 +135117,10 @@ define <16 x double> @bitcast_v64f16_to_v16f64(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136688,10 +136662,10 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-LABEL: bitcast_v16f64_to_v64i16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136705,7 +136679,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; VI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; VI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; VI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -136716,16 +136689,15 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; VI-NEXT: .LBB84_2: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16f64_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -136739,7 +136711,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
@@ -136750,7 +136721,6 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
; GFX9-NEXT: .LBB84_2: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v16f64_to_v64i16:
@@ -138227,10 +138197,10 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v16f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -216217,10 +216187,10 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -225990,10 +225960,10 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v64bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -227982,10 +227952,10 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64f16_to_v64i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
@@ -230347,10 +230317,10 @@ define <64 x half> @bitcast_v64i16_to_v64f16(<64 x i16> %a, i32 %b) #0 {
; GFX9-LABEL: bitcast_v64i16_to_v64f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index fd469c3f069bf..e212efdfbbb1e 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2753,42 +2753,46 @@ define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {
; SDAG-LABEL: load_v32i8:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: buffer_load_dwordx4 v[36:39], off, s[16:19], 0
-; SDAG-NEXT: buffer_load_dwordx4 v[32:35], off, s[16:19], 0 offset:16
+; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0
+; SDAG-NEXT: buffer_load_dwordx4 v[16:19], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_lshrrev_b64 v[19:20], 24, v[32:33]
-; SDAG-NEXT: v_lshrrev_b64 v[27:28], 24, v[34:35]
-; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v36
-; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v36
-; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v37
-; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v38
-; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v38
-; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v39
-; SDAG-NEXT: v_lshrrev_b32_e32 v17, 8, v32
-; SDAG-NEXT: v_lshrrev_b32_e32 v18, 16, v32
-; SDAG-NEXT: v_lshrrev_b32_e32 v21, 8, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v22, 16, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v23, 24, v33
-; SDAG-NEXT: v_lshrrev_b32_e32 v25, 8, v34
-; SDAG-NEXT: v_lshrrev_b32_e32 v26, 16, v34
-; SDAG-NEXT: v_lshrrev_b32_e32 v29, 8, v35
-; SDAG-NEXT: v_lshrrev_b32_e32 v30, 16, v35
-; SDAG-NEXT: v_lshrrev_b32_e32 v31, 24, v35
-; SDAG-NEXT: v_mov_b32_e32 v0, v36
-; SDAG-NEXT: v_mov_b32_e32 v4, v37
-; SDAG-NEXT: v_mov_b32_e32 v8, v38
-; SDAG-NEXT: v_mov_b32_e32 v12, v39
-; SDAG-NEXT: v_mov_b32_e32 v16, v32
-; SDAG-NEXT: v_mov_b32_e32 v20, v33
-; SDAG-NEXT: v_mov_b32_e32 v24, v34
-; SDAG-NEXT: v_mov_b32_e32 v28, v35
+; SDAG-NEXT: v_lshrrev_b64 v[33:34], 24, v[0:1]
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_lshrrev_b64 v[34:35], 24, v[16:17]
+; SDAG-NEXT: v_lshrrev_b32_e32 v38, 8, v0
+; SDAG-NEXT: v_lshrrev_b32_e32 v32, 16, v0
+; SDAG-NEXT: v_lshrrev_b32_e32 v36, 8, v16
+; SDAG-NEXT: v_lshrrev_b32_e32 v37, 16, v16
+; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[2:3]
+; SDAG-NEXT: v_lshrrev_b64 v[27:28], 24, v[18:19]
+; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; SDAG-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; SDAG-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; SDAG-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; SDAG-NEXT: v_mov_b32_e32 v4, v1
+; SDAG-NEXT: v_mov_b32_e32 v8, v2
+; SDAG-NEXT: v_mov_b32_e32 v12, v3
+; SDAG-NEXT: v_mov_b32_e32 v20, v17
+; SDAG-NEXT: v_mov_b32_e32 v24, v18
+; SDAG-NEXT: v_mov_b32_e32 v28, v19
+; SDAG-NEXT: v_mov_b32_e32 v3, v33
+; SDAG-NEXT: v_mov_b32_e32 v19, v34
+; SDAG-NEXT: v_mov_b32_e32 v1, v38
+; SDAG-NEXT: v_mov_b32_e32 v2, v32
+; SDAG-NEXT: v_mov_b32_e32 v17, v36
+; SDAG-NEXT: v_mov_b32_e32 v18, v37
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: load_v32i8:
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 56415ef44c527..64c512a64ca50 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -626,12 +626,12 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, 44
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[18:21], v[16:17]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[22:25], v[12:13]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[14:15]
; GFX942-SDAG-NEXT: ; kill: killed $vgpr12_vgpr13
-; GFX942-SDAG-NEXT: ; kill: killed $vgpr14_vgpr15
; GFX942-SDAG-NEXT: ; kill: killed $vgpr16_vgpr17
; GFX942-SDAG-NEXT: s_nop 0
-; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[8:9]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[14:15]
+; GFX942-SDAG-NEXT: s_nop 0
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[8:9]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[30:33], v[10:11]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[48:51], v[6:7]
@@ -650,10 +650,10 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[6:7]
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[26:29] offset:32
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[48:51]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[52:55]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir b/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
index 0c9c5ad84c092..efb97b70fdacd 100644
--- a/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
+++ b/llvm/test/CodeGen/AMDGPU/live-interval-bug-in-rename-independent-subregs.mir
@@ -14,12 +14,10 @@ body: |
; REG_ALLOC-NEXT: renamable $vgpr11_vgpr12_vgpr13_vgpr14 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr3, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr2, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $vgpr5_vgpr6_vgpr7_vgpr8 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr0, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; REG_ALLOC-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr4, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; REG_ALLOC-NEXT: KILL killed renamable $vgpr4
; REG_ALLOC-NEXT: KILL killed renamable $vgpr2
; REG_ALLOC-NEXT: KILL killed renamable $vgpr0
; REG_ALLOC-NEXT: KILL killed renamable $vgpr3
- ; REG_ALLOC-NEXT: KILL killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
+ ; REG_ALLOC-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr4, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; REG_ALLOC-NEXT: renamable $sgpr8 = V_READFIRSTLANE_B32 killed $vgpr11, implicit $exec
; REG_ALLOC-NEXT: renamable $sgpr9 = V_READFIRSTLANE_B32 killed $vgpr12, implicit $exec
; REG_ALLOC-NEXT: renamable $sgpr6_sgpr7 = V_CMP_NE_U32_e64 killed $vgpr1, 0, implicit $exec
@@ -83,12 +81,10 @@ body: |
; DEAD_INST_DEL-NEXT: renamable $vgpr11_vgpr12_vgpr13_vgpr14 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr3, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr2, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $vgpr5_vgpr6_vgpr7_vgpr8 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr0, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; DEAD_INST_DEL-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr4, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
- ; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr4
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr2
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr0
; DEAD_INST_DEL-NEXT: KILL killed renamable $vgpr3
- ; DEAD_INST_DEL-NEXT: KILL killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
+ ; DEAD_INST_DEL-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr4, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
; DEAD_INST_DEL-NEXT: renamable $sgpr8 = V_READFIRSTLANE_B32 killed $vgpr11, implicit $exec
; DEAD_INST_DEL-NEXT: renamable $sgpr9 = V_READFIRSTLANE_B32 killed $vgpr12, implicit $exec
; DEAD_INST_DEL-NEXT: renamable $sgpr6_sgpr7 = V_CMP_NE_U32_e64 killed $vgpr1, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.mir b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
index e3e2a5297b125..1375d036978c6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
@@ -424,9 +424,10 @@ body: |
# GCN-LABEL: {{^}}name: ptr_use_after_clause_subreg_multi{{$}}
# GCN: dead %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub0_sub1, 0, 0, implicit $exec
# GCN-NEXT: dead %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub2_sub3, 16, 0, implicit $exec
+# GCN-NEXT: KILL %0.sub2{{$}}
# GCN-NEXT: dead %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub3_sub4, 32, 0, implicit $exec
# GCN-NEXT: dead %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0.sub6_sub7, 48, 0, implicit $exec
-# GCN-NEXT: KILL %0.sub2_sub3_sub4, %0.sub7{{$}}
+# GCN-NEXT: KILL %0.sub3_sub4, %0.sub7{{$}}
# GCN-NEXT: S_NOP 0, implicit %0.sub0_sub1, implicit %0.sub5_sub6
---
name: ptr_use_after_clause_subreg_multi
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
index c22e48557b876..0542fb8c99ca4 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
@@ -197,6 +197,8 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: ; def s21
; GFX906-NEXT: ;;#ASMEND
; GFX906-NEXT: v_writelane_b32 v39, s21, 12
+; GFX906-NEXT: s_or_saveexec_b64 s[34:35], -1
+; GFX906-NEXT: s_mov_b64 exec, s[34:35]
; GFX906-NEXT: ;;#ASMSTART
; GFX906-NEXT: ; def s22
; GFX906-NEXT: ;;#ASMEND
@@ -237,10 +239,10 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: v_readlane_b32 s8, v39, 6
; GFX906-NEXT: v_readlane_b32 s10, v39, 4
; GFX906-NEXT: v_readlane_b32 s16, v39, 22
+; GFX906-NEXT: v_mov_b32_e32 v31, v40
; GFX906-NEXT: v_readlane_b32 s12, v39, 3
; GFX906-NEXT: v_readlane_b32 s13, v39, 2
; GFX906-NEXT: v_readlane_b32 s14, v39, 1
-; GFX906-NEXT: v_mov_b32_e32 v31, v40
; GFX906-NEXT: v_readlane_b32 s15, v39, 0
; GFX906-NEXT: v_readlane_b32 s5, v39, 11
; GFX906-NEXT: v_readlane_b32 s7, v39, 9
@@ -579,6 +581,8 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: ; def s21
; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: v_writelane_b32 v39, s21, 12
+; GFX908-NEXT: s_or_saveexec_b64 s[34:35], -1
+; GFX908-NEXT: s_mov_b64 exec, s[34:35]
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def s22
; GFX908-NEXT: ;;#ASMEND
@@ -619,10 +623,10 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: v_readlane_b32 s8, v39, 6
; GFX908-NEXT: v_readlane_b32 s10, v39, 4
; GFX908-NEXT: v_readlane_b32 s16, v39, 22
+; GFX908-NEXT: v_mov_b32_e32 v31, v40
; GFX908-NEXT: v_readlane_b32 s12, v39, 3
; GFX908-NEXT: v_readlane_b32 s13, v39, 2
; GFX908-NEXT: v_readlane_b32 s14, v39, 1
-; GFX908-NEXT: v_mov_b32_e32 v31, v40
; GFX908-NEXT: v_readlane_b32 s15, v39, 0
; GFX908-NEXT: v_readlane_b32 s5, v39, 11
; GFX908-NEXT: v_readlane_b32 s7, v39, 9
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
index 516db40a5af76..f4c098ec0464e 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-early-clobber.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
# Test that the upward register pressure tracker accounts for early-clobber
# physical register defs overlapping with physical register uses.
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
index bc6f1e1888bbc..476ab04e2a120 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-physreg-limits.mir
@@ -1,7 +1,7 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 %s 2>&1 | FileCheck %s --check-prefix=RPU
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 %s 2>&1 | FileCheck %s --check-prefix=RPU-NOPHYS
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward %s 2>&1 | FileCheck %s --check-prefix=RPD
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-trackers-physical-register-tracking=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=null --run-pass=amdgpu-print-rp -amdgpu-use-amdgpu-trackers=1 -amdgpu-print-rp-downward -amdgpu-track-physregs-in-gcn-trackers=0 %s 2>&1 | FileCheck %s --check-prefix=RPD-NOPHYS
# Tests for physical register pressure tracking edge cases:
# 1. Non-allocatable registers (e.g. $scc) should not affect pressure.
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
index be5f45ff4fcea..f0c5f9e59bf6b 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir
@@ -161,7 +161,7 @@ body: |
; RPU: bb.0:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 0 1
; RPU-NEXT: 0 1 %0:vgpr_32 = COPY $vgpr0
; RPU-NEXT: 0 1
; RPU-NEXT: 0 3 %1:vreg_64 = IMPLICIT_DEF
@@ -272,8 +272,8 @@ body: |
; RPD: bb.0:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 %0:vgpr_32 = COPY $vgpr0
+ ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 0 2 %0:vgpr_32 = COPY $vgpr0
; RPD-NEXT: 0 1
; RPD-NEXT: 0 3 %1:vreg_64 = IMPLICIT_DEF
; RPD-NEXT: 0 3
@@ -498,35 +498,35 @@ body: |
; RPU: bb.0:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0 = COPY $sgpr1
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr2_sgpr3 = S_GETPC_B64
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr1 = COPY killed $sgpr3
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 S_BRANCH %bb.1
- ; RPU-NEXT: 0 1
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 0 $sgpr0 = COPY $sgpr1
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 2 0 $sgpr2_sgpr3 = S_GETPC_B64
+ ; RPU-NEXT: 2 0
+ ; RPU-NEXT: 2 0 $sgpr1 = COPY killed $sgpr3
+ ; RPU-NEXT: 2 0
+ ; RPU-NEXT: 4 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 4 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 S_BRANCH %bb.1
+ ; RPU-NEXT: 1 1
; RPU-NEXT: Live-out: %0:0000000000000C00
; RPU-NEXT: Live-thr:
; RPU-NEXT: 0 0
; RPU-NEXT: bb.1:
; RPU-NEXT: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
- ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPU-NEXT: 0 1
- ; RPU-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPU-NEXT: 1 1
+ ; RPU-NEXT: 1 1 $m0 = S_MOV_B32 killed $sgpr0
; RPU-NEXT: 0 1
; RPU-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
; RPU-NEXT: 0 1
@@ -551,38 +551,38 @@ body: |
; RPD: bb.0:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0 = COPY $sgpr1
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr2_sgpr3 = S_GETPC_B64
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr1 = COPY killed $sgpr3
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 S_BRANCH %bb.1
- ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 2 0 $sgpr0 = COPY $sgpr1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 3 0 $sgpr2_sgpr3 = S_GETPC_B64
+ ; RPD-NEXT: 3 0
+ ; RPD-NEXT: 4 0 $sgpr1 = COPY killed $sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 4 0 $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM $sgpr0_sgpr1, 0, 0
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 4 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0
+ ; RPD-NEXT: 3 0
+ ; RPD-NEXT: 3 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_CBRANCH_SCC1 %bb.2, implicit $scc
+ ; RPD-NEXT: 3 1
+ ; RPD-NEXT: 3 1 S_BRANCH %bb.1
+ ; RPD-NEXT: 3 1
; RPD-NEXT: Live-out: %0:0000000000000C00
; RPD-NEXT: Live-thr:
; RPD-NEXT: 0 0
; RPD-NEXT: bb.1:
; RPD-NEXT: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0
- ; RPD-NEXT: 0 1
- ; RPD-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
- ; RPD-NEXT: 0 1
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 1 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec
+ ; RPD-NEXT: 1 1
+ ; RPD-NEXT: 1 1 $m0 = S_MOV_B32 killed $sgpr0
+ ; RPD-NEXT: 1 1
+ ; RPD-NEXT: 1 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec
+ ; RPD-NEXT: 1 1
; RPD-NEXT: Live-out: %0:0000000000000C00
; RPD-NEXT: Live-thr:
; RPD-NEXT: 0 0
@@ -632,7 +632,7 @@ body: |
; RPU-LABEL: name: test_partially_used_def
; RPU: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %1:sgpr_128 = COPY %0:sgpr_128
@@ -646,8 +646,8 @@ body: |
; RPD-LABEL: name: test_partially_used_def
; RPD: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPD-NEXT: 4 0
; RPD-NEXT: 8 0 %1:sgpr_128 = COPY %0:sgpr_128
; RPD-NEXT: 1 0
@@ -666,19 +666,33 @@ tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr0_sgpr1_sgpr2_sgpr3
- ; RP-LABEL: name: test_partially_used_early_clobber_def
- ; RP: Live-in:
- ; RP-NEXT: SGPR VGPR
- ; RP-NEXT: 0 0
- ; RP-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
- ; RP-NEXT: 4 0
- ; RP-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
- ; RP-NEXT: 1 0
- ; RP-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
- ; RP-NEXT: 0 0
- ; RP-NEXT: Live-out:
- ; RP-NEXT: Live-thr:
- ; RP-NEXT: 0 0
+ ; RPU-LABEL: name: test_partially_used_early_clobber_def
+ ; RPU: Live-in:
+ ; RPU-NEXT: SGPR VGPR
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPU-NEXT: 4 0
+ ; RPU-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
+ ; RPU-NEXT: 1 0
+ ; RPU-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
+ ; RPU-NEXT: 0 0
+ ; RPU-NEXT: Live-out:
+ ; RPU-NEXT: Live-thr:
+ ; RPU-NEXT: 0 0
+ ;
+ ; RPD-LABEL: name: test_partially_used_early_clobber_def
+ ; RPD: Live-in:
+ ; RPD-NEXT: SGPR VGPR
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128
+ ; RPD-NEXT: 1 0
+ ; RPD-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128
+ ; RPD-NEXT: 0 0
+ ; RPD-NEXT: Live-out:
+ ; RPD-NEXT: Live-thr:
+ ; RPD-NEXT: 0 0
%0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
early-clobber %1:sgpr_128 = COPY %0:sgpr_128
S_NOP 0, implicit %1.sub1
@@ -692,7 +706,7 @@ body: |
; RPU-LABEL: name: test_partially_used_def_and_early_clobber_def
; RPU: Live-in:
; RPU-NEXT: SGPR VGPR
- ; RPU-NEXT: 0 0
+ ; RPU-NEXT: 4 0
; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPU-NEXT: 4 0
; RPU-NEXT: 16 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128
@@ -706,8 +720,8 @@ body: |
; RPD-LABEL: name: test_partially_used_def_and_early_clobber_def
; RPD: Live-in:
; RPD-NEXT: SGPR VGPR
- ; RPD-NEXT: 0 0
- ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ ; RPD-NEXT: 4 0
+ ; RPD-NEXT: 8 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; RPD-NEXT: 4 0
; RPD-NEXT: 20 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128
; RPD-NEXT: 6 0
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
index d6e3b6a734eec..906547512f8f4 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveins.mir
@@ -3,7 +3,7 @@
# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
-# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 \
# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
diff --git a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
index 0be216846f4be..1d84311e385f7 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-physreg-liveouts.mir
@@ -3,7 +3,7 @@
# RUN: -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler \
# RUN: -filetype=null %s 2>&1 | FileCheck --check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=machine-scheduler \
-# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 \
+# RUN: -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 \
# RUN: -debug-only=machine-scheduler -filetype=null %s 2>&1 \
# RUN: | FileCheck --check-prefix=GCN-NOPHYS %s
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index b2b73e9a96fcb..f54ed99c5035d 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,6 +1,6 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index 91a071dd69049..47de1b20f6703 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -15,18 +15,19 @@
;
; Check scheduling pressure values:
; SCHED-LABEL: spill:%bb.0 entry
-; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 98
-; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 97
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
+; SCHED: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
;
; SCHED-GCNTRACKERS-LABEL: spill:%bb.0 entry
; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 193
; SCHED-GCNTRACKERS: Pressure after scheduling: VGPRs: 0 AGPRs: 0, SGPRs: 98
;
-; NOTE: GCN Trackers now track pressure from both virtual and physical registers.
-; The GCN tracker now matches the generic tracker's VGPR count (1 VGPR).
-; When a live range is not found for a physical regunit, we conservatively
-; assume the unit is live, so Region SGPR pressure can be higher (193 vs 98).
-; Pressure after scheduling remains 98 vs 97 due to physical register tracking.
+; NOTE: Physical register pressure tracking is now decoupled from the GCN
+; trackers, so both the generic and GCN-tracker scheduling paths account for
+; physical registers. As a result SCHED and SCHED-GCNTRACKERS now report the
+; same pressure. When a live range is not found for a physical regunit, we
+; conservatively assume the unit is live, so Region SGPR pressure is higher
+; (193) than the virtual-only count (98).
define amdgpu_kernel void @spill(ptr addrspace(1) %arg, i32 %cnd) #0 {
entry:
@@ -267,7 +268,7 @@ bb3:
; GCN-GCNTRACKERS: ScratchSize: 8
;
; SCHED-LABEL: spill_func:%bb.0 entry
-; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 97
+; SCHED: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
;
; SCHED-GCNTRACKERS-LABEL: spill_func:%bb.0 entry
; SCHED-GCNTRACKERS: Region register pressure: VGPRs: 0 AGPRs: 0, SGPRs: 192
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
index 7c947f4a942a6..cc7ad68fc6546 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-gcn-physreg-pressure.ll
@@ -3,7 +3,7 @@
; RUN: FileCheck --check-prefix=GCN-DEBUG %s < %t
; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=NO-GCN %s
; RUN: FileCheck --check-prefix=GENERIC-DEBUG %s < %t
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-trackers-physical-register-tracking=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-NOPHYS %s
; RUN: FileCheck --check-prefix=GCN-NOPHYS-DEBUG %s < %t
; REQUIRES: asserts
@@ -14,8 +14,8 @@
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_single_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_single_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -73,7 +73,7 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_multiple_physregs
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_multiple_physregs
@@ -138,8 +138,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
; GENERIC-DEBUG-LABEL: test_physreg_with_vreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 7, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 10, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 2 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 12
; GCN-NOPHYS-DEBUG-LABEL: test_physreg_with_vreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 2 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 12
@@ -217,8 +217,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_early_clobber
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -277,8 +277,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GENERIC-DEBUG-LABEL: test_early_clobber_tuple
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 9, LVGPR WT: 0, LSGPR WT: 9
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GCN-NOPHYS-DEBUG-LABEL: test_early_clobber_tuple
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
@@ -347,8 +347,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GENERIC-DEBUG-LABEL: test_physreg_input
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
; GCN-NOPHYS-DEBUG-LABEL: test_physreg_input
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 5, LVGPR WT: 0, LSGPR WT: 6
@@ -407,8 +407,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GENERIC-DEBUG-LABEL: test_tuple_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
; GCN-NOPHYS-DEBUG-LABEL: test_tuple_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 6, LVGPR WT: 0, LSGPR WT: 6
@@ -468,8 +468,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
; GENERIC-DEBUG-LABEL: test_tuple128_physreg
-; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
+; GENERIC-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 1 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 12
; GCN-NOPHYS-DEBUG-LABEL: test_tuple128_physreg
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 1 AGPRs: 0, SGPRs: 8, LVGPR WT: 0, LSGPR WT: 8
@@ -528,8 +528,8 @@ entry:
; GCN-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
; GENERIC-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
-; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
-; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 12, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 16, LVGPR WT: 0, LSGPR WT: 16
+; GENERIC-DEBUG: Pressure after scheduling: VGPRs: 3 AGPRs: 0, SGPRs: 13, LVGPR WT: 0, LSGPR WT: 16
; GCN-NOPHYS-DEBUG-LABEL: test_vreg_and_physreg_live_range_overlap
; GCN-NOPHYS-DEBUG: Region register pressure: VGPRs: 3 AGPRs: 0, SGPRs: 14, LVGPR WT: 0, LSGPR WT: 16
diff --git a/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
index 13c17b415f6d7..1b18cfb698081 100644
--- a/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
+++ b/llvm/test/CodeGen/AMDGPU/soft-clause-dbg-value.mir
@@ -28,7 +28,6 @@ body: |
; CHECK-NEXT: DBG_VALUE [[S_LOAD_DWORD_IMM2]], 0, 0
; CHECK-NEXT: DBG_VALUE [[S_LOAD_DWORD_IMM3]], 0, 0
; CHECK-NEXT: [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 64, 0 :: (load (s32), addrspace 4)
- ; CHECK-NEXT: KILL [[COPY]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[S_LOAD_DWORD_IMM]], implicit [[S_LOAD_DWORD_IMM1]], implicit [[S_LOAD_DWORD_IMM2]], implicit [[S_LOAD_DWORD_IMM3]], implicit [[S_LOAD_DWORD_IMM4]]
%0:sreg_64 = COPY $sgpr4_sgpr5
%1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (load (s32), align 4, addrspace 4)
>From b5d4a1bfb3e8f928c36c8c3228c748dc696a52e8 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 18 Aug 2026 17:54:20 -0500
Subject: [PATCH 3/5] Passed additional required argument.
---
llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
index 62d97dd3db818..c176a23f646fb 100644
--- a/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
+++ b/llvm/unittests/Target/AMDGPU/GCNRegPressureTest.cpp
@@ -185,7 +185,7 @@ body: |
MachineInstr *U1 = Instrs[2];
MachineInstr *U2 = Instrs[3];
- GCNDownwardRPTracker RPTracker(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MRI);
GCNRPTracker::LiveRegSet Empty;
RPTracker.reset(MRI, Empty);
@@ -236,7 +236,7 @@ body: |
MachineInstr *DefS0 = Instrs[0];
MachineInstr *UseRedef = Instrs[1];
- GCNDownwardRPTracker RPTracker(LIS);
+ GCNDownwardRPTracker RPTracker(LIS, MRI);
GCNRPTracker::LiveRegSet Empty;
RPTracker.reset(MRI, Empty);
>From f96542af2a040854e88355e826c16ee2362311d5 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 18 Aug 2026 19:51:48 -0500
Subject: [PATCH 4/5] Fix test churn.
---
.../AMDGPU/agpr-copy-no-free-registers.ll | 2 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 188 +++++++++---------
.../CodeGen/AMDGPU/call-argument-types.ll | 12 +-
.../AMDGPU/materialize-frame-index-sgpr.ll | 12 +-
.../schedule-amdgpu-tracker-physreg-crash.ll | 9 +-
.../AMDGPU/splitkit-getsubrangeformask.ll | 9 -
6 files changed, 114 insertions(+), 118 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index d6b7d1eb83187..8a07f883e7a66 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -1256,9 +1256,9 @@ define void @no_free_vgprs_at_sgpr_to_agpr_copy(float %v0, float %v1) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_accvgpr_write_b32 a2, v2
; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX90A-NEXT: v_accvgpr_write_b32 a2, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1
; GFX90A-NEXT: v_accvgpr_write_b32 a3, v3
; GFX90A-NEXT: v_accvgpr_write_b32 a4, v4
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 24cebc1097205..66dacf22fcb87 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -13346,10 +13346,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[0:1], v[38:39], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[48:49], 8
; SI-NEXT: v_lshr_b64 v[19:20], v[36:37], 24
-; SI-NEXT: v_lshr_b64 v[32:33], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[36:37], 16
; SI-NEXT: v_lshr_b64 v[17:18], v[36:37], 8
; SI-NEXT: v_lshr_b64 v[11:12], v[34:35], 24
-; SI-NEXT: v_lshr_b64 v[28:29], v[34:35], 16
+; SI-NEXT: v_lshr_b64 v[32:33], v[34:35], 16
; SI-NEXT: v_lshr_b64 v[9:10], v[34:35], 8
; SI-NEXT: v_lshr_b64 v[3:4], v[38:39], 24
; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 8
@@ -13391,10 +13391,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v24, s42
; SI-NEXT: v_mov_b32_e32 v25, s44
; SI-NEXT: v_mov_b32_e32 v19, s24
-; SI-NEXT: v_mov_b32_e32 v32, s26
+; SI-NEXT: v_mov_b32_e32 v28, s26
; SI-NEXT: v_mov_b32_e32 v17, s28
; SI-NEXT: v_mov_b32_e32 v11, s10
-; SI-NEXT: v_mov_b32_e32 v28, s12
+; SI-NEXT: v_mov_b32_e32 v32, s12
; SI-NEXT: v_mov_b32_e32 v9, s14
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v0, s6
@@ -13403,10 +13403,10 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v38
; SI-NEXT: v_mov_b32_e32 v4, v39
-; SI-NEXT: v_mov_b32_e32 v10, v28
+; SI-NEXT: v_mov_b32_e32 v10, v32
; SI-NEXT: v_mov_b32_e32 v8, v34
; SI-NEXT: v_mov_b32_e32 v12, v35
-; SI-NEXT: v_mov_b32_e32 v18, v32
+; SI-NEXT: v_mov_b32_e32 v18, v28
; SI-NEXT: v_mov_b32_e32 v16, v36
; SI-NEXT: v_mov_b32_e32 v20, v37
; SI-NEXT: v_mov_b32_e32 v26, v24
@@ -26885,14 +26885,14 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI-NEXT: s_cmp_lg_u32 s5, 1
; VI-NEXT: s_cbranch_scc1 .LBB85_5
; VI-NEXT: ; %bb.4: ; %cmp.true
+; VI-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; VI-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
-; VI-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
-; VI-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; VI-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
+; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
-; VI-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
; VI-NEXT: v_lshrrev_b32_e32 v31, 24, v25
; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
@@ -27022,14 +27022,14 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: s_cmp_lg_u32 s5, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; GFX9-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
-; GFX9-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
-; GFX9-NEXT: v_add_f64 v[24:25], s[22:23], 1.0
+; GFX9-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
+; GFX9-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX9-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
-; GFX9-NEXT: v_lshrrev_b64 v[27:28], 24, v[24:25]
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 24, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 8, v25
@@ -38420,88 +38420,88 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s79
; SI-NEXT: v_cvt_f32_f16_e32 v1, s22
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s78
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s20
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
; SI-NEXT: v_or_b32_e32 v33, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s78
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s23
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s77
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
-; SI-NEXT: v_bfe_u32 v31, v30, 8, 8
-; SI-NEXT: v_or_b32_e32 v34, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s77
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s20
-; SI-NEXT: v_lshr_b64 v[24:25], v[33:34], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[33:34], 24
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[25:26], v[33:34], 8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v34
-; SI-NEXT: v_or_b32_e32 v35, v1, v0
+; SI-NEXT: v_or_b32_e32 v34, v2, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s76
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v35, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s21
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s75
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v22, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_bfe_u32 v23, v22, 8, 8
-; SI-NEXT: v_or_b32_e32 v36, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s75
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s18
-; SI-NEXT: v_lshr_b64 v[19:20], v[35:36], 24
-; SI-NEXT: v_lshr_b64 v[20:21], v[35:36], 16
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[17:18], v[35:36], 8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v36
-; SI-NEXT: v_or_b32_e32 v37, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s74
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s19
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
+; SI-NEXT: v_or_b32_e32 v36, v0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s74
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s19
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
-; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
-; SI-NEXT: v_or_b32_e32 v38, v1, v0
+; SI-NEXT: v_or_b32_e32 v37, v2, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s73
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_or_b32_e32 v38, v1, v2
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_lshr_b64 v[11:12], v[37:38], 24
-; SI-NEXT: v_lshr_b64 v[12:13], v[37:38], 16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s72
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshr_b64 v[9:10], v[37:38], 8
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
; SI-NEXT: v_or_b32_e32 v48, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s72
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
-; SI-NEXT: v_or_b32_e32 v49, v1, v0
+; SI-NEXT: v_or_b32_e32 v49, v2, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[37:38], 24
+; SI-NEXT: v_lshr_b64 v[16:17], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[33:34], 16
; SI-NEXT: v_lshr_b64 v[3:4], v[48:49], 24
; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 8
+; SI-NEXT: v_lshr_b64 v[12:13], v[37:38], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[37:38], 8
+; SI-NEXT: v_lshr_b64 v[19:20], v[35:36], 24
+; SI-NEXT: v_lshr_b64 v[17:18], v[35:36], 8
+; SI-NEXT: v_lshr_b64 v[27:28], v[33:34], 24
+; SI-NEXT: v_lshr_b64 v[25:26], v[33:34], 8
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v49
+; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
+; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v36
+; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v34
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
+; SI-NEXT: v_bfe_u32 v23, v22, 8, 8
+; SI-NEXT: v_bfe_u32 v31, v30, 8, 8
; SI-NEXT: s_branch .LBB105_6
; SI-NEXT: .LBB105_5:
; SI-NEXT: v_mov_b32_e32 v30, s78
@@ -38531,7 +38531,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s14
; SI-NEXT: v_mov_b32_e32 v9, s24
; SI-NEXT: v_mov_b32_e32 v19, s28
-; SI-NEXT: v_mov_b32_e32 v20, s40
+; SI-NEXT: v_mov_b32_e32 v16, s40
; SI-NEXT: v_mov_b32_e32 v17, s42
; SI-NEXT: v_mov_b32_e32 v27, s46
; SI-NEXT: v_mov_b32_e32 v24, s56
@@ -38543,7 +38543,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, v12
; SI-NEXT: v_mov_b32_e32 v8, v37
; SI-NEXT: v_mov_b32_e32 v12, v38
-; SI-NEXT: v_mov_b32_e32 v18, v20
+; SI-NEXT: v_mov_b32_e32 v18, v16
; SI-NEXT: v_mov_b32_e32 v16, v35
; SI-NEXT: v_mov_b32_e32 v20, v36
; SI-NEXT: v_mov_b32_e32 v26, v24
@@ -42517,57 +42517,57 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_or_b32_e32 v6, 0x400000, v2
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_lshrrev_b64 v[24:25], 16, v[5:6]
+; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v25, v28
+; VI-NEXT: v_lshrrev_b64 v[24:25], 16, v[5:6]
; VI-NEXT: v_mov_b32_e32 v1, v4
; VI-NEXT: v_mov_b32_e32 v9, v12
; VI-NEXT: v_mov_b32_e32 v17, v20
-; VI-NEXT: v_lshrrev_b64 v[36:37], 24, v[24:25]
-; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[16:17]
-; VI-NEXT: v_lshrrev_b64 v[34:35], 24, v[8:9]
-; VI-NEXT: v_lshrrev_b64 v[32:33], 24, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v25, v28
+; VI-NEXT: v_lshrrev_b64 v[50:51], 24, v[24:25]
+; VI-NEXT: v_lshrrev_b64 v[48:49], 24, v[16:17]
+; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[8:9]
+; VI-NEXT: v_lshrrev_b64 v[34:35], 24, v[0:1]
; VI-NEXT: v_lshrrev_b32_e32 v31, 24, v28
; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v28
; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v28
; VI-NEXT: v_lshrrev_b32_e32 v26, 16, v24
-; VI-NEXT: v_lshrrev_b32_e32 v25, 8, v24
+; VI-NEXT: v_lshrrev_b32_e32 v32, 8, v24
; VI-NEXT: v_lshrrev_b32_e32 v23, 24, v20
; VI-NEXT: v_lshrrev_b32_e32 v22, 16, v20
; VI-NEXT: v_lshrrev_b32_e32 v21, 8, v20
; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; VI-NEXT: v_lshrrev_b32_e32 v17, 8, v16
+; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v16
; VI-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v12
; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v12
; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; VI-NEXT: v_lshrrev_b32_e32 v36, 8, v8
; VI-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; VI-NEXT: v_lshrrev_b32_e32 v39, 8, v0
; VI-NEXT: s_branch .LBB109_6
; VI-NEXT: .LBB109_5:
; VI-NEXT: v_mov_b32_e32 v26, s59
-; VI-NEXT: v_mov_b32_e32 v25, s58
+; VI-NEXT: v_mov_b32_e32 v32, s58
; VI-NEXT: v_mov_b32_e32 v31, s57
; VI-NEXT: v_mov_b32_e32 v30, s56
; VI-NEXT: v_mov_b32_e32 v29, s47
; VI-NEXT: v_mov_b32_e32 v18, s46
-; VI-NEXT: v_mov_b32_e32 v17, s45
+; VI-NEXT: v_mov_b32_e32 v33, s45
; VI-NEXT: v_mov_b32_e32 v23, s44
; VI-NEXT: v_mov_b32_e32 v22, s43
; VI-NEXT: v_mov_b32_e32 v21, s42
; VI-NEXT: v_mov_b32_e32 v10, s41
-; VI-NEXT: v_mov_b32_e32 v9, s40
+; VI-NEXT: v_mov_b32_e32 v36, s40
; VI-NEXT: v_mov_b32_e32 v15, s29
; VI-NEXT: v_mov_b32_e32 v14, s28
; VI-NEXT: v_mov_b32_e32 v13, s27
; VI-NEXT: v_mov_b32_e32 v2, s26
-; VI-NEXT: v_mov_b32_e32 v1, s25
+; VI-NEXT: v_mov_b32_e32 v39, s25
; VI-NEXT: v_mov_b32_e32 v7, s24
; VI-NEXT: v_mov_b32_e32 v6, s15
; VI-NEXT: v_mov_b32_e32 v5, s14
@@ -42579,15 +42579,19 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v12, s19
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v4, s17
-; VI-NEXT: v_mov_b32_e32 v36, s10
-; VI-NEXT: v_mov_b32_e32 v37, s8
-; VI-NEXT: v_mov_b32_e32 v34, s6
-; VI-NEXT: v_mov_b32_e32 v32, s4
+; VI-NEXT: v_mov_b32_e32 v50, s10
+; VI-NEXT: v_mov_b32_e32 v48, s8
+; VI-NEXT: v_mov_b32_e32 v37, s6
+; VI-NEXT: v_mov_b32_e32 v34, s4
; VI-NEXT: .LBB109_6: ; %end
-; VI-NEXT: v_mov_b32_e32 v3, v32
-; VI-NEXT: v_mov_b32_e32 v11, v34
-; VI-NEXT: v_mov_b32_e32 v19, v37
-; VI-NEXT: v_mov_b32_e32 v27, v36
+; VI-NEXT: v_mov_b32_e32 v3, v34
+; VI-NEXT: v_mov_b32_e32 v11, v37
+; VI-NEXT: v_mov_b32_e32 v19, v48
+; VI-NEXT: v_mov_b32_e32 v27, v50
+; VI-NEXT: v_mov_b32_e32 v1, v39
+; VI-NEXT: v_mov_b32_e32 v9, v36
+; VI-NEXT: v_mov_b32_e32 v17, v33
+; VI-NEXT: v_mov_b32_e32 v25, v32
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: bitcast_v16bf16_to_v32i8_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index b5e4c0d7fad3b..4abbbc99a296c 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -5334,11 +5334,11 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
@@ -5567,11 +5567,11 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
@@ -6905,11 +6905,11 @@ define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val
; GISEL-NEXT: v_mov_b32_e32 v14, s50
; GISEL-NEXT: v_mov_b32_e32 v15, s51
; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
+; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v17, s9
; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: v_mov_b32_e32 v19, s11
-; GISEL-NEXT: s_mov_b64 s[0:1], s[52:53]
-; GISEL-NEXT: s_mov_b64 s[2:3], s[54:55]
; GISEL-NEXT: v_mov_b32_e32 v20, s12
; GISEL-NEXT: v_mov_b32_e32 v21, s13
; GISEL-NEXT: v_mov_b32_e32 v22, s14
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index a76318434a391..0b1babb8438d8 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX900 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX942 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 --amdgpu-xnack=true < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX942 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10_1 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10_3 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX7-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_1-GCNTRACKERS %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_3-GCNTRACKERS %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX11-GCNTRACKERS %s
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index f54ed99c5035d..8d60cdd236146 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack 2>&1 < %s | FileCheck -check-prefixes=GCN %s
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
@@ -18,8 +18,9 @@
}
; GCN-TRACKERS-NOT: ran out of registers during register allocation
+; GCN-TRACKERS-NOT: unhandled SGPR spill to memory
; GCN-NOT: ran out of registers during register allocation
-; GCN-NOPHYS-FAIL: ran out of registers during register allocation
+; GCN-NOPHYS-FAIL: unhandled SGPR spill to memory
; GCN Trackers now track physical register pressure correctly, so this test
; verifies that both trackers can successfully handle code with heavy physical
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index d4a4f6ed9eefa..1366efba7da32 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -46,8 +46,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_]], 16, 0 :: (invariant load (s128) from %ir.83, addrspace 4)
; CHECK-NEXT: early-clobber %67:sgpr_256 = S_LOAD_DWORDX8_IMM_ec undef %68:sreg_64, 0, 0 :: (invariant load (s256) from `ptr addrspace(4) poison`, align 16, addrspace 4)
- ; CHECK-NEXT: KILL [[S_ADD_U32_]].sub0, [[S_ADD_U32_]].sub1
- ; CHECK-NEXT: KILL undef %68:sreg_64
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_LOAD_DWORDX4_IMM]], 0, 0 :: (dereferenceable invariant load (s32) from %ir.84, align 1, addrspace 8)
; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -57,7 +55,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %88:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], %67.sub0_sub1_sub2_sub3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL undef %88:sgpr_128
; CHECK-NEXT: [[S_SUB_I32_2:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM1]], 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_1:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
@@ -204,8 +201,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_LOAD_DWORD_IMM1:%[0-9]+]].sub0:sgpr_128 = S_LOAD_DWORD_IMM [[S_ADD_U32_21]], 168, 0 :: (invariant load (s32) from %ir.308, align 8, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM22]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM23]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM22]]
; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]].sub2:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]].sub2
; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]].sub3:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]].sub3
; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]].sub1:sgpr_128 = S_AND_B32 %67.sub0, 65535, implicit-def dead $scc
@@ -228,10 +223,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[V_MOV_B32_e32_]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM26]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM24]]
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM25]]
; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -2, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -1, [[BUFFER_LOAD_FORMAT_X_IDXEN1]], 0, implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -3, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
>From 5f1755fe60e0444149d3642120bb123354cd8f3c Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Wed, 19 Aug 2026 12:58:14 -0500
Subject: [PATCH 5/5] triple migration.
---
.../AMDGPU/materialize-frame-index-sgpr.ll | 32 +++++++++----------
.../schedule-amdgpu-tracker-physreg-crash.ll | 6 ++--
.../AMDGPU/schedule-amdgpu-tracker-physreg.ll | 4 +--
3 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index 0b1babb8438d8..22831bed332ee 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 --amdgpu-xnack=true < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX900 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX942 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10_1 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10_3 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX7-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_1-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_3-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX11-GCNTRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX12-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -mtriple=amdgpu8.10-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX900 %s
+; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --amdgpu-xnack=true < %s | FileCheck -check-prefixes=GFX942 %s
+; RUN: llc -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10_1 %s
+; RUN: llc -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefix=GFX10_3 %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX7-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu8.10-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX8-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX900-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefixes=GFX942-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_1-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX10_3-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX11-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 < %s | FileCheck -check-prefix=GFX12-GCNTRACKERS %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index 8d60cdd236146..f2996e452b88f 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true 2>&1 < %s | FileCheck -check-prefixes=GCN %s
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 2>&1 < %s | FileCheck -check-prefixes=GCN-TRACKERS %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true 2>&1 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -amdgpu-track-physregs-in-gcn-trackers=0 2>&1 < %s | FileCheck --check-prefix=GCN-NOPHYS-FAIL %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
<16 x i32>, <7 x i32>, ; vgprs
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index 47de1b20f6703..28d32f8fc6476 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu6.00-amd-amdhsa -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN %s
; RUN: FileCheck --check-prefix=SCHED %s < %t
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-GCNTRACKERS %s
+; RUN: llc -mtriple=amdgpu6.00-amd-amdhsa -amdgpu-s-branch-bits=5 -amdgpu-long-branch-factor=0 -amdgpu-use-amdgpu-trackers=1 -debug-only=machine-scheduler < %s 2> %t | FileCheck --check-prefix=GCN-GCNTRACKERS %s
; RUN: FileCheck --check-prefix=SCHED-GCNTRACKERS %s < %t
; REQUIRES: asserts
; CHECK-LABEL: {{^}}spill:
More information about the llvm-branch-commits
mailing list