[llvm] [AMDGPU] Improved register pressure estimates for rescheduling (PR #213116)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 14:09:53 PDT 2026
https://github.com/ilia-cher updated https://github.com/llvm/llvm-project/pull/213116
>From bfa01ad9634f2cd8f357e46cb3b90570bb490daf Mon Sep 17 00:00:00 2001
From: Ilya Chernyavsky <ichernia at amd.com>
Date: Thu, 30 Jul 2026 18:58:13 +0000
Subject: [PATCH] [AMDGPU] Improved register pressure estimates for
rescheduling
Currently we use instantaneous register pressure estimates during
scheduling. This simple metric counts the number of registers live at
any given moment; scheduler uses it to determine when we are over
the register budget and should start prioritizing instructions that
free registers.
Unfortunately, this metric can sometimes be overly optimistic: for
example, it does not account for register redefinitions (common for
loop-carried vregs), and can underestimate the true register pressure.
This can often result in unnecessary spills and loss of performance in
production kernels.
To fix this, we introduce a stage that selectively reschedules certain
regions: it uses a lightweight, greedy, non-splitting register
allocation simulation to detect when the instantaneous estimate might
be wrong and a region should be rescheduled with tighter register
limits.
Test plan:
ninja -C build
build/bin/llvm-lit -v llvm/test/CodeGen/AMDGPU/lirp.mir
+ perf testing on a corpus of kernels
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 7 +
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 86 ++++++++++-
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 15 ++
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 139 +++++++++++++++---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 37 ++++-
llvm/test/CodeGen/AMDGPU/lirp.mir | 55 +++++++
6 files changed, 311 insertions(+), 28 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/lirp.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 726be8c7f0982..b3b86a776fb39 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -21,6 +21,9 @@ using namespace llvm::AMDGPU;
#define DEBUG_TYPE "machine-scheduler"
+// Default VGPR threshold percent for coexec scheduler.
+static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
+
namespace {
// Used to disable post-RA scheduling with function level granularity.
@@ -412,10 +415,14 @@ AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
const MachineSchedContext *C)
: GCNSchedStrategy(C) {
SchedStages.push_back(GCNSchedStageID::ILPInitialSchedule);
+ SchedStages.push_back(GCNSchedStageID::LiveIntervalRPReschedule);
SchedStages.push_back(GCNSchedStageID::RewriteMFMAForm);
SchedStages.push_back(GCNSchedStageID::PreRARematerialize);
// Use more accurate GCN pressure trackers.
UseGCNTrackers = true;
+
+ if (!VGPRThresholdPercentOpt.getNumOccurrences())
+ VGPRThresholdPercent = DefaultCoExecVGPRThresholdPercent;
}
void AMDGPUCoExecSchedStrategy::initPolicy(MachineBasicBlock::iterator Begin,
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 53617e89af757..dd1988deb0065 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -14,6 +14,8 @@
#include "GCNRegPressure.h"
#include "AMDGPU.h"
#include "SIMachineFunctionInfo.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/LiveIntervalUnion.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/RegisterPressure.h"
@@ -651,8 +653,8 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
MachineBasicBlock::const_iterator End,
const LiveRegSet *LiveRegsCopy) {
MBBEnd = MI.getParent()->end();
- assert(End == MBBEnd ||
- End->getParent()->end() == MBBEnd && "end unrelated to MI block");
+ assert((End == MBBEnd || End->getParent()->end() == MBBEnd) &&
+ "end unrelated to MI block");
NextMI = &MI;
NextMI = skipDebugInstructionsForward(NextMI, End);
@@ -1213,3 +1215,83 @@ LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,
}
}
#endif
+
+unsigned llvm::estimateGreedyVGPRPressure(
+ MachineBasicBlock::const_iterator RegionBegin,
+ MachineBasicBlock::const_iterator RegionEnd,
+ const GCNRPTracker::LiveRegSet &LiveIns, const LiveIntervals &LIS,
+ const MachineRegisterInfo &MRI, const SIRegisterInfo &TRI) {
+
+ auto IsVGPR = [&MRI, &TRI](Register VReg) {
+ const TargetRegisterClass *RC = MRI.getRegClass(VReg);
+ return TRI.isVGPRClass(RC) || TRI.isVectorSuperClass(RC);
+ };
+
+ SetVector<const LiveInterval *> IntervalSet;
+ IntervalSet.reserve(LiveIns.size());
+
+ // Collect live-ins
+ for (const auto &[RegNum, LaneMask] : LiveIns) {
+ Register VReg(RegNum);
+ if (!VReg.isVirtual() || !LIS.hasInterval(VReg) || !IsVGPR(VReg))
+ continue;
+ const LiveInterval &LI = LIS.getInterval(VReg);
+ IntervalSet.insert(&LI);
+ }
+
+ // Collect defs in region
+ for (MachineBasicBlock::const_iterator I = RegionBegin; I != RegionEnd; ++I) {
+ for (const MachineOperand &MO : I->operands()) {
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ Register VReg = MO.getReg();
+ if (!VReg.isVirtual() || !LIS.hasInterval(VReg) || !IsVGPR(VReg))
+ continue;
+ const LiveInterval &LI = LIS.getInterval(VReg);
+ IntervalSet.insert(&LI);
+ }
+ }
+
+ SmallVector<const LiveInterval *> Intervals = IntervalSet.takeVector();
+ llvm::sort(Intervals, [](const LiveInterval *LHS, const LiveInterval *RHS) {
+ return LHS->beginIndex() < RHS->beginIndex();
+ });
+
+ LiveIntervalUnion::Allocator Alloc;
+ std::vector<LiveIntervalUnion> Slots;
+ unsigned MaxSlotUsed = 0;
+
+ // Simulate greedy register allocation, assuming unlimited number
+ // of physical registers (slots).
+ for (const LiveInterval *LI : Intervals) {
+ const TargetRegisterClass *RC = MRI.getRegClass(LI->reg());
+ unsigned Width = TRI.getRegClassWeight(RC).RegWeight;
+ unsigned Alignment = std::max(1u, TRI.getRegClassAlignmentNumBits(RC) / 32);
+
+ unsigned Start = 0;
+ while (true) {
+ unsigned End = Start + Width;
+ if (Slots.size() < End)
+ Slots.resize(End, LiveIntervalUnion(Alloc));
+
+ bool Fits = true;
+ for (unsigned Idx = Start; Idx < End; Idx++) {
+ LiveIntervalUnion::Query Q(*LI, Slots[Idx]);
+ if (Q.checkInterference()) {
+ Start = alignTo(Idx + 1, Alignment);
+ Fits = false;
+ break;
+ }
+ }
+
+ if (Fits) {
+ for (unsigned Idx = Start; Idx < End; Idx++)
+ Slots[Idx].unify(*LI, *LI);
+ MaxSlotUsed = std::max(MaxSlotUsed, End);
+ break;
+ }
+ }
+ }
+
+ return MaxSlotUsed;
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index df0bfd1a0cc5d..2f413fca61070 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -592,6 +592,21 @@ LLVM_ABI void dumpMaxRegPressure(MachineFunction &MF,
LiveIntervals &LIS,
const MachineLoopInfo *MLI);
+/// Estimate VGPR pressure using greedy, non-splitting register allocation
+/// simulation, accounting for live interval interference.
+/// \param RegionBegin Start iterator of the region
+/// \param RegionEnd End iterator of the region
+/// \param LiveIns Live-in registers for the region
+/// \param LIS LiveIntervals analysis
+/// \param MRI MachineRegisterInfo
+/// \param TRI Target register info
+/// \returns estimated VGPR pressure
+unsigned estimateGreedyVGPRPressure(
+ MachineBasicBlock::const_iterator RegionBegin,
+ MachineBasicBlock::const_iterator RegionEnd,
+ const GCNRPTracker::LiveRegSet &LiveIns, const LiveIntervals &LIS,
+ const MachineRegisterInfo &MRI, const SIRegisterInfo &TRI);
+
} // end namespace llvm
#endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 5816559fcc899..a24ded20ee7bb 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -104,25 +104,18 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(true));
-namespace {
+bool VGPRThresholdParser::parse(cl::Option &O, StringRef ArgName, StringRef Arg,
+ unsigned &Value) {
+ if (Arg.getAsInteger(0, Value))
+ return O.error("'" + Arg + "' value invalid for uint argument!");
-struct VGPRThresholdParser : public cl::parser<unsigned> {
- VGPRThresholdParser(cl::Option &O) : cl::parser<unsigned>(O) {}
+ if (Value > 100)
+ return O.error("'" + Arg + "' value must be in the range [0, 100]!");
- bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
- if (Arg.getAsInteger(0, Value))
- return O.error("'" + Arg + "' value invalid for uint argument!");
-
- if (Value > 100)
- return O.error("'" + Arg + "' value must be in the range [0, 100]!");
-
- return false;
- }
-};
-
-} // end anonymous namespace
+ return false;
+}
-static cl::opt<unsigned, false, VGPRThresholdParser> VGPRThresholdPercentOpt(
+cl::opt<unsigned, false, VGPRThresholdParser> llvm::VGPRThresholdPercentOpt(
"amdgpu-vgpr-threshold-percent", cl::Hidden,
cl::desc("Percent of VGPR limits that we should use as RP threshold "
"during scheduling. We have two limits relevant to scheduling: "
@@ -138,6 +131,7 @@ GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
DownwardTracker(*C->LIS), UpwardTracker(*C->LIS), HasHighPressure(false) {
if (GCNTrackers.getNumOccurrences() > 0)
GCNTrackersOverride = GCNTrackers;
+ VGPRThresholdPercent = VGPRThresholdPercentOpt;
}
void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
@@ -183,14 +177,13 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
VGPRCriticalLimit = std::min(VGPRBudget, VGPRExcessLimit);
}
// Apply VGPR excess threshold percentage if specified.
- if (VGPRThresholdPercentOpt > 0) {
+ if (VGPRThresholdPercent > 0) {
[[maybe_unused]] unsigned OriginalVGPRExcessLimit = VGPRExcessLimit;
[[maybe_unused]] unsigned OriginalVGPRCriticalLimit = VGPRCriticalLimit;
- VGPRExcessLimit = (VGPRThresholdPercentOpt * VGPRExcessLimit + 99) / 100;
- VGPRCriticalLimit =
- (VGPRThresholdPercentOpt * VGPRCriticalLimit + 99) / 100;
+ VGPRExcessLimit = (VGPRThresholdPercent * VGPRExcessLimit + 99) / 100;
+ VGPRCriticalLimit = (VGPRThresholdPercent * VGPRCriticalLimit + 99) / 100;
LLVM_DEBUG(dbgs() << "Applied VGPR excess threshold "
- << VGPRThresholdPercentOpt << "%, VGPRExcessLimit: "
+ << VGPRThresholdPercent << "%, VGPRExcessLimit: "
<< OriginalVGPRExcessLimit << " -> " << VGPRExcessLimit
<< ". VGPRCriticalLimit: " << OriginalVGPRCriticalLimit
<< " -> " << VGPRCriticalLimit << '\n');
@@ -1055,6 +1048,8 @@ GCNScheduleDAGMILive::createSchedStage(GCNSchedStageID SchedStageID) {
case GCNSchedStageID::MemoryClauseInitialSchedule:
return std::make_unique<MemoryClauseInitialScheduleStage>(SchedStageID,
*this);
+ case GCNSchedStageID::LiveIntervalRPReschedule:
+ return std::make_unique<LiveIntervalRPStage>(SchedStageID, *this);
}
llvm_unreachable("Unknown SchedStageID.");
@@ -1297,6 +1292,9 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const GCNSchedStageID &StageID) {
case GCNSchedStageID::MemoryClauseInitialSchedule:
OS << "Max memory clause Initial Schedule";
break;
+ case GCNSchedStageID::LiveIntervalRPReschedule:
+ OS << "Live Interval RP Reschedule";
+ break;
}
return OS;
@@ -2235,6 +2233,105 @@ bool MemoryClauseInitialScheduleStage::shouldRevertScheduling(
return mayCauseSpilling(WavesAfter);
}
+static cl::opt<bool> EnableLiveIntervalRPReschedule(
+ "amdgpu-lirp-reschedule", cl::Hidden,
+ cl::desc("Enable live interval RP reschedule stage"), cl::init(true));
+
+static cl::opt<unsigned> LiveIntervalRPThreshold(
+ "amdgpu-lirp-threshold", cl::Hidden,
+ cl::desc("Percent increase of live interval RP over instant pressure to "
+ "trigger rescheduling"),
+ cl::init(10));
+
+static cl::opt<unsigned> LiveIntervalRPVGPRReduction(
+ "amdgpu-lirp-vgpr-reduction", cl::Hidden,
+ cl::desc(
+ "Reduction factor (percent) for VGPR threshold during live interval RP "
+ "reschedule stage"),
+ cl::init(90));
+
+static cl::opt<unsigned> LiveIntervalRPVGPRReductionEpilogue(
+ "amdgpu-lirp-vgpr-reduction-epilogue", cl::Hidden,
+ cl::desc(
+ "Reduction factor (percent) for VGPR threshold during live interval RP "
+ "reschedule stage for exit blocks"),
+ cl::init(70));
+
+bool LiveIntervalRPStage::initGCNSchedStage() {
+ if (!EnableLiveIntervalRPReschedule)
+ return false;
+
+ if (!GCNSchedStage::initGCNSchedStage())
+ return false;
+
+ if (!S.VGPRThresholdPercent) {
+ LLVM_DEBUG(dbgs() << "LIRP: expected VGPRThresholdPercent to be enabled, "
+ "not using live interval RP reschedule stage\n");
+ return false;
+ }
+
+ return true;
+}
+
+bool LiveIntervalRPStage::initGCNRegion() {
+ unsigned InstantRP = DAG.Pressure[RegionIdx].getArchVGPRNum();
+ auto [RegionBegin, RegionEnd] = DAG.Regions[RegionIdx];
+ if (RegionBegin == RegionEnd)
+ return false;
+
+ unsigned LIRP = estimateGreedyVGPRPressure(
+ RegionBegin, RegionEnd, DAG.LiveIns[RegionIdx], *DAG.getLIS(),
+ DAG.MF.getRegInfo(), static_cast<const SIRegisterInfo &>(*DAG.TRI));
+
+ bool IsReturnBlock = RegionBegin->getParent()->isReturnBlock();
+ // Use a tighter VGPRExcessLimit by reducing VGPRThresholdPercent
+ unsigned BlockReduction = IsReturnBlock ? LiveIntervalRPVGPRReductionEpilogue
+ : LiveIntervalRPVGPRReduction;
+ unsigned NewVGPRThresholdPercent =
+ (S.VGPRThresholdPercent * BlockReduction + 99) / 100;
+
+ LLVM_DEBUG(dbgs() << "LIRP: Region " << RegionIdx
+ << ", VGPRThresholdPercent: " << S.VGPRThresholdPercent
+ << " -> " << NewVGPRThresholdPercent
+ << ", VGPRExcessLimit=" << S.VGPRExcessLimit
+ << ", VGPRCriticalLimit=" << S.VGPRCriticalLimit
+ << ", InstantRP=" << InstantRP << ", LIRP=" << LIRP);
+
+ bool DoRescheduling = false;
+ // Lower bound on InstantRP to skip over tiny regions
+ unsigned InstantRPLB = S.VGPRExcessLimit / 10;
+ if (LIRP > S.VGPRExcessLimit) {
+ LLVM_DEBUG(dbgs() << " [LIRP exceeds the limit (" << S.VGPRExcessLimit
+ << "), rescheduling]");
+ DoRescheduling = true;
+ } else if (LIRP > InstantRP && InstantRP > InstantRPLB) {
+ unsigned IncreasePercent = ((LIRP - InstantRP) * 100) / InstantRP;
+ if (IncreasePercent > LiveIntervalRPThreshold) {
+ LLVM_DEBUG(dbgs() << " [" << IncreasePercent << "% > "
+ << LiveIntervalRPThreshold << "%, rescheduling]");
+ DoRescheduling = true;
+ }
+ }
+ LLVM_DEBUG(dbgs() << '\n');
+
+ if (DoRescheduling && GCNSchedStage::initGCNRegion()) {
+ SavedVGPRExcessLimit = S.VGPRExcessLimit;
+ SavedVGPRCriticalLimit = S.VGPRCriticalLimit;
+ SavedVGPRThresholdPercent = S.VGPRThresholdPercent;
+ S.VGPRThresholdPercent = NewVGPRThresholdPercent;
+ return true;
+ }
+
+ return false;
+}
+
+void LiveIntervalRPStage::finalizeGCNRegion() {
+ S.VGPRExcessLimit = SavedVGPRExcessLimit;
+ S.VGPRCriticalLimit = SavedVGPRCriticalLimit;
+ S.VGPRThresholdPercent = SavedVGPRThresholdPercent;
+ GCNSchedStage::finalizeGCNRegion();
+}
+
bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
if (WavesAfter <= MFI.getMinWavesPerEU() && isRegionWithExcessRP() &&
!PressureAfter.less(MF, PressureBefore)) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2059f4e6479ff..91ca01da2a162 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -20,9 +20,17 @@
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/Rematerializer.h"
+#include "llvm/Support/CommandLine.h"
namespace llvm {
+struct VGPRThresholdParser : public cl::parser<unsigned> {
+ VGPRThresholdParser(cl::Option &O) : cl::parser<unsigned>(O) {}
+ bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value);
+};
+
+extern cl::opt<unsigned, false, VGPRThresholdParser> VGPRThresholdPercentOpt;
+
class SIMachineFunctionInfo;
class SIRegisterInfo;
class GCNSubtarget;
@@ -35,7 +43,8 @@ enum class GCNSchedStageID : unsigned {
ClusteredLowOccupancyReschedule = 3,
PreRARematerialize = 4,
ILPInitialSchedule = 5,
- MemoryClauseInitialSchedule = 6
+ MemoryClauseInitialSchedule = 6,
+ LiveIntervalRPReschedule = 7
};
#ifndef NDEBUG
@@ -88,10 +97,6 @@ class GCNSchedStrategy : public GenericScheduler {
std::vector<unsigned> MaxPressure;
- unsigned SGPRExcessLimit;
-
- unsigned VGPRExcessLimit;
-
unsigned TargetOccupancy;
MachineFunction *MF;
@@ -132,6 +137,10 @@ class GCNSchedStrategy : public GenericScheduler {
// Bias for VGPR limits under a high register pressure.
const unsigned HighRPVGPRBias = 7;
+ unsigned SGPRExcessLimit;
+
+ unsigned VGPRExcessLimit;
+
unsigned SGPRCriticalLimit;
unsigned VGPRCriticalLimit;
@@ -140,6 +149,8 @@ class GCNSchedStrategy : public GenericScheduler {
unsigned VGPRLimitBias = 0;
+ unsigned VGPRThresholdPercent = 0;
+
GCNSchedStrategy(const MachineSchedContext *C);
SUnit *pickNode(bool &IsTopNode) override;
@@ -265,6 +276,7 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
friend class ClusteredLowOccStage;
friend class PreRARematStage;
friend class ILPInitialScheduleStage;
+ friend class LiveIntervalRPStage;
friend class RegionPressureMap;
const GCNSubtarget &ST;
@@ -797,6 +809,21 @@ class MemoryClauseInitialScheduleStage : public GCNSchedStage {
: GCNSchedStage(StageID, DAG) {}
};
+class LiveIntervalRPStage : public GCNSchedStage {
+public:
+ bool initGCNSchedStage() override;
+ bool initGCNRegion() override;
+ void finalizeGCNRegion() override;
+
+ LiveIntervalRPStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
+ : GCNSchedStage(StageID, DAG) {}
+
+private:
+ unsigned SavedVGPRThresholdPercent = 0;
+ unsigned SavedVGPRExcessLimit = 0;
+ unsigned SavedVGPRCriticalLimit = 0;
+};
+
class GCNPostScheduleDAGMILive final : public ScheduleDAGMI {
private:
std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
diff --git a/llvm/test/CodeGen/AMDGPU/lirp.mir b/llvm/test/CodeGen/AMDGPU/lirp.mir
new file mode 100644
index 0000000000000..9bee5bd7d8f59
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lirp.mir
@@ -0,0 +1,55 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -amdgpu-lirp-reschedule=true -debug-only=machine-scheduler %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -amdgpu-lirp-reschedule=false -debug-only=machine-scheduler %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=DISABLED
+# REQUIRES: asserts
+
+# Test that the live interval pressure reschedule stage runs and computes pressure for regions.
+
+# CHECK: Starting scheduling stage: Live Interval RP Reschedule
+# CHECK: LIRP: Region 0,{{.*}}InstantRP=2, LIRP=3
+# DISABLED-NOT: LIRP:
+---
+name: lirp_estimate_test
+tracksRegLiveness: true
+isSSA: false
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; A (%0) has two segments with a gap
+ ; B (%1) overlaps A seg1 and extends into the gap
+ ; C (%2) is defined in the gap and overlaps A seg2
+ ; B and C overlap in the gap
+ ;
+ ; InstantRP: max 2 at any point
+ ; LIRP: 3 because A, B, C all interfere with each other
+
+ %0:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec ; A seg1
+ %1:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec ; B def
+ BUFFER_STORE_DWORD_OFFEN %0, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec ; use A (seg1 ends)
+ ; gap in A
+ %2:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec ; C def (B and C overlap)
+ BUFFER_STORE_DWORD_OFFEN %1, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec ; use B (dies)
+ ; end of gap
+ %0:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec ; A seg2
+ BUFFER_STORE_DWORD_OFFEN %0, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec ; use A (seg2 dies)
+ BUFFER_STORE_DWORD_OFFEN %2, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec ; use C (dies)
+ S_ENDPGM 0
+...
+
+# CHECK: LIRP: Region 0,{{.*}}InstantRP=1, LIRP=1
+---
+name: lirp_slot_reuse_test
+tracksRegLiveness: true
+isSSA: false
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; %0 and %1 have disjoint live ranges, so RA can reuse the same slot.
+ ; InstantRP: max 1 (only one value live at a time)
+ ; LIRP: 1 (%1 reuses %0's slot after %0 dies)
+
+ %0:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %0, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec ; %0 dies
+ %1:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %1, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec ; %1 dies
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list