[llvm-branch-commits] [llvm] [AMDGPU][NFC] Track banked VGPR peaks in the GCN RP tracker (PR #243967)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 10:57:49 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Dhruva Chakrabarti (dhruvachak)
<details>
<summary>Changes</summary>
Keep banked VGPR peaks in GCNRPTracker alongside the maximum pressure, for each requested set of bank limits, instead of computing them in pressure callbacks in the scheduler. getRealRegPressure now returns the peaks for the limits it is given.
Depends on https://github.com/llvm/llvm-project/pull/243965
---
Full diff: https://github.com/llvm/llvm-project/pull/243967.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/GCNRegPressure.cpp (+4-2)
- (modified) llvm/lib/Target/AMDGPU/GCNRegPressure.h (+54-10)
- (modified) llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp (+32-44)
- (modified) llvm/lib/Target/AMDGPU/GCNSchedStrategy.h (+6-6)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index dbb9d7bc8aacc..2506b0d7f420e 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -588,7 +588,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
this->MRI = &MRI;
LastTrackedMI = nullptr;
LiveRegs = llvm::getLiveRegs(SI, LIS, MRI);
- MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
+ CurPressure = getRegPressure(MRI, LiveRegs);
+ resetMaxPressure();
}
void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
@@ -597,7 +598,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
LastTrackedMI = nullptr;
if (&this->LiveRegs != &LiveRegs)
this->LiveRegs = LiveRegs;
- MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
+ CurPressure = getRegPressure(MRI, LiveRegs);
+ resetMaxPressure();
}
/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 8ba9af94fd650..ec8426e7125d5 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -116,19 +116,34 @@ struct GCNRegPressure {
}
/// Peak ArchVGPR32 and AccVGPR32 pressure over a set of program points, after
- /// distributing the AVGPR pressure of each point with getBankedVGPRNum().
+ /// distributing the AVGPR pressure of each point with getBankedVGPRNum()
+ /// against the limits of the peaks.
struct BankedVGPRPeaks {
+ unsigned ArchVGPRLimit = 0;
+ unsigned AGPRLimit = 0;
unsigned ArchVGPRs = 0;
unsigned AGPRs = 0;
+ /// Peak combined-file pressure of a single point.
+ unsigned PointUnifiedVGPRs = 0;
- void add(const GCNRegPressure &RP, unsigned ArchLimit, unsigned AccLimit) {
- auto [Arch, Acc] = RP.getBankedVGPRNum(ArchLimit, AccLimit);
+ BankedVGPRPeaks() = default;
+ BankedVGPRPeaks(unsigned ArchLimit, unsigned AccLimit)
+ : ArchVGPRLimit(ArchLimit), AGPRLimit(AccLimit) {}
+
+ void add(const GCNRegPressure &RP) {
+ auto [Arch, Acc] = RP.getBankedVGPRNum(ArchVGPRLimit, AGPRLimit);
ArchVGPRs = std::max(ArchVGPRs, Arch);
AGPRs = std::max(AGPRs, Acc);
+ PointUnifiedVGPRs =
+ std::max(PointUnifiedVGPRs, getBankedUnifiedVGPRNum(Arch, Acc));
}
- /// \returns the combined-file pressure. The banks use separate ranges of
- /// the file, so this is the sum of both peaks.
+ /// Clears the peaks, keeping the limits.
+ void clear() { ArchVGPRs = AGPRs = PointUnifiedVGPRs = 0; }
+
+ /// \returns the combined-file pressure of the two peaks. The banks use
+ /// separate ranges of the file, so this is the sum of both peaks, which can
+ /// exceed PointUnifiedVGPRs.
unsigned getUnifiedVGPRNum() const {
return getBankedUnifiedVGPRNum(ArchVGPRs, AGPRs);
}
@@ -401,17 +416,27 @@ class GCNRPTracker {
MachineBasicBlock::const_iterator LastTrackedMI;
mutable const MachineRegisterInfo *MRI = nullptr;
std::function<void(const GCNRegPressure &)> PressureCallback;
+ /// Banked VGPR peaks of the points folded into MaxPressure, one per set of
+ /// bank limits.
+ SmallVector<GCNRegPressure::BankedVGPRPeaks, 2> BankedPeaks;
GCNRPTracker(LiveIntervals &LIS_) : LIS(LIS_) {}
- /// Folds the pressure \p RP of one program point into MaxPressure and passes
- /// it to the pressure callback.
+ /// Folds the pressure \p RP of one program point into MaxPressure and the
+ /// banked peaks, and passes it to the pressure callback.
void updateMaxPressure(const GCNRegPressure &RP) {
MaxPressure = maxPerRegFile(MaxPressure, RP);
+ for (GCNRegPressure::BankedVGPRPeaks &Peaks : BankedPeaks)
+ Peaks.add(RP);
if (PressureCallback)
PressureCallback(RP);
}
+ void clearBankedPeaks() {
+ for (GCNRegPressure::BankedVGPRPeaks &Peaks : BankedPeaks)
+ Peaks.clear();
+ }
+
/// Resets tracker before or \p After the provided \p MI, which can be a debug
/// instruction.
void reset(const MachineInstr &MI, bool After);
@@ -434,11 +459,30 @@ class GCNRPTracker {
return LastTrackedMI.isValid() ? &*LastTrackedMI : nullptr;
}
- void clearMaxPressure() { MaxPressure.clear(); }
+ void clearMaxPressure() {
+ MaxPressure.clear();
+ clearBankedPeaks();
+ }
const GCNRegPressure &getMaxPressure() const { return MaxPressure; }
- void resetMaxPressure() { MaxPressure = CurPressure; }
+ void resetMaxPressure() {
+ MaxPressure = CurPressure;
+ clearBankedPeaks();
+ for (GCNRegPressure::BankedVGPRPeaks &Peaks : BankedPeaks)
+ Peaks.add(CurPressure);
+ }
+
+ /// Makes the tracker also keep banked VGPR peaks, one for the bank limits of
+ /// each element of \p Peaks. Set this before resetting the tracker.
+ void trackBankedPeaks(ArrayRef<GCNRegPressure::BankedVGPRPeaks> Peaks) {
+ BankedPeaks.assign(Peaks.begin(), Peaks.end());
+ clearBankedPeaks();
+ }
+
+ ArrayRef<GCNRegPressure::BankedVGPRPeaks> getBankedPeaks() const {
+ return BankedPeaks;
+ }
/// Sets a function that the tracker calls with the pressure of every program
/// point that it folds into the maximum pressure. A user can compute there a
@@ -544,7 +588,7 @@ class GCNDownwardRPTracker : public GCNDownwardRPTrackerBase {
/// \p return MaxPressure and clear it.
GCNRegPressure moveMaxPressure() {
auto Res = MaxPressure;
- MaxPressure.clear();
+ clearMaxPressure();
return Res;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 2bf51596644d6..1f02ea2d9c814 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1059,24 +1059,21 @@ void GCNScheduleDAGMILive::schedule() {
GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure(
unsigned RegionIdx,
- std::function<void(const GCNRegPressure &)> PressureCallback) const {
+ MutableArrayRef<GCNRegPressure::BankedVGPRPeaks> BankedPeaks) const {
if (Regions[RegionIdx].first == Regions[RegionIdx].second) {
GCNRegPressure RP = llvm::getRegPressure(MRI, LiveIns[RegionIdx]);
- if (PressureCallback)
- PressureCallback(RP);
+ for (GCNRegPressure::BankedVGPRPeaks &Peaks : BankedPeaks) {
+ Peaks.clear();
+ Peaks.add(RP);
+ }
return RP;
}
GCNDownwardRPTracker RPTracker(*LIS);
- bool HasInstrs =
- RPTracker.reset(*Regions[RegionIdx].first, Regions[RegionIdx].second,
- &LiveIns[RegionIdx]);
- // reset() does not pass the live-in pressure to the callback.
- if (PressureCallback) {
- PressureCallback(RPTracker.getPressure());
- RPTracker.setPressureCallback(std::move(PressureCallback));
- }
- if (HasInstrs)
+ RPTracker.trackBankedPeaks(BankedPeaks);
+ if (RPTracker.reset(*Regions[RegionIdx].first, Regions[RegionIdx].second,
+ &LiveIns[RegionIdx]))
RPTracker.advance(Regions[RegionIdx].second);
+ llvm::copy(RPTracker.getBankedPeaks(), BankedPeaks.begin());
return RPTracker.moveMaxPressure();
}
@@ -1640,17 +1637,14 @@ bool GCNSchedStage::initGCNRegion() {
BankedPeaksBefore = {};
if (useBankedSpillCheck()) {
- unsigned ArchLimit, AccLimit;
- std::tie(ArchLimit, AccLimit) = getSpillCheckBankLimits();
+ auto [ArchLimit, AccLimit] = getSpillCheckBankLimits();
+ BankedPeaksBefore = {ArchLimit, AccLimit};
// If no point has AVGPRs beyond the ArchVGPR bank, banking the region
// maximum is exact. Otherwise bank each point.
- if (PressureBefore.getArchVGPRNum() <= ArchLimit) {
- BankedPeaksBefore.add(PressureBefore, ArchLimit, AccLimit);
- } else {
- DAG.getRealRegPressure(RegionIdx, [&](const GCNRegPressure &RP) {
- BankedPeaksBefore.add(RP, ArchLimit, AccLimit);
- });
- }
+ if (PressureBefore.getArchVGPRNum() <= ArchLimit)
+ BankedPeaksBefore.add(PressureBefore);
+ else
+ DAG.getRealRegPressure(RegionIdx, BankedPeaksBefore);
}
LLVM_DEBUG(
@@ -1778,30 +1772,24 @@ void GCNSchedStage::checkScheduling() {
// distributing it across the banks can miss a rigid peak at another point.
// Distribute the pressure of each point instead.
bool AVGPRBanking = ST.getCLOpts().avgpr_banking;
- unsigned ArchVGPRBankLimit, AGPRBankLimit;
- std::tie(ArchVGPRBankLimit, AGPRBankLimit) = S.getBankLimits();
- unsigned BankedArchVGPRPeak = 0, BankedAGPRPeak = 0, BankedUnifiedPeak = 0;
- BankedPeaksAfter = {};
bool BankedSpillCheck = useBankedSpillCheck();
- unsigned SpillCheckArchLimit, SpillCheckAccLimit;
- std::tie(SpillCheckArchLimit, SpillCheckAccLimit) = getSpillCheckBankLimits();
- std::function<void(const GCNRegPressure &)> TrackBankedPeaks;
- if (AVGPRBanking) {
- TrackBankedPeaks = [&](const GCNRegPressure &RP) {
- auto [ArchNum, AccNum] =
- RP.getBankedVGPRNum(ArchVGPRBankLimit, AGPRBankLimit);
- BankedArchVGPRPeak = std::max(BankedArchVGPRPeak, ArchNum);
- BankedAGPRPeak = std::max(BankedAGPRPeak, AccNum);
- BankedUnifiedPeak =
- std::max(BankedUnifiedPeak,
- GCNRegPressure::getBankedUnifiedVGPRNum(ArchNum, AccNum));
- if (BankedSpillCheck)
- BankedPeaksAfter.add(RP, SpillCheckArchLimit, SpillCheckAccLimit);
- };
+ auto [ArchVGPRBankLimit, AGPRBankLimit] = S.getBankLimits();
+ // The excess checks below use the bank limits, while mayCauseSpilling() uses
+ // the spill check limits.
+ SmallVector<GCNRegPressure::BankedVGPRPeaks, 2> BankedPeaks;
+ if (AVGPRBanking)
+ BankedPeaks.emplace_back(ArchVGPRBankLimit, AGPRBankLimit);
+ if (BankedSpillCheck) {
+ auto [ArchLimit, AccLimit] = getSpillCheckBankLimits();
+ BankedPeaks.emplace_back(ArchLimit, AccLimit);
}
// Check the results of scheduling.
- PressureAfter = DAG.getRealRegPressure(RegionIdx, TrackBankedPeaks);
+ PressureAfter = DAG.getRealRegPressure(RegionIdx, BankedPeaks);
+ GCNRegPressure::BankedVGPRPeaks ExcessCheckPeaks =
+ AVGPRBanking ? BankedPeaks.front() : GCNRegPressure::BankedVGPRPeaks();
+ BankedPeaksAfter =
+ BankedSpillCheck ? BankedPeaks.back() : GCNRegPressure::BankedVGPRPeaks();
LLVM_DEBUG(dbgs() << "Pressure after scheduling: " << print(PressureAfter));
LLVM_DEBUG(dbgs() << "Region: " << RegionIdx << ".\n");
@@ -1862,13 +1850,13 @@ void GCNSchedStage::checkScheduling() {
unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
bool BankExcess = AVGPRBanking
- ? BankedArchVGPRPeak > ArchVGPRBankLimit ||
- BankedAGPRPeak > AGPRBankLimit
+ ? ExcessCheckPeaks.ArchVGPRs > ArchVGPRBankLimit ||
+ ExcessCheckPeaks.AGPRs > AGPRBankLimit
: PressureAfter.getArchVGPRNum() > MaxArchVGPRs ||
PressureAfter.getAGPRNum() > MaxArchVGPRs;
unsigned VGPRNum = AVGPRBanking && UnifiedVGPRFile
- ? BankedUnifiedPeak
+ ? ExcessCheckPeaks.PointUnifiedVGPRs
: PressureAfter.getVGPRNum(UnifiedVGPRFile);
if (VGPRNum > MaxVGPRs || BankExcess ||
PressureAfter.getSGPRNum() > MaxSGPRs) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 0972168e07a50..d8239ecc74f8c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -360,12 +360,12 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
RegionPressureMap RegionLiveOuts;
// Return current region pressure.
- /// \returns the maximum pressure of region \p RegionIdx. If set, \p
- /// PressureCallback is called with the pressure of every program point of
- /// the region.
- GCNRegPressure getRealRegPressure(unsigned RegionIdx,
- std::function<void(const GCNRegPressure &)>
- PressureCallback = nullptr) const;
+ /// \returns the maximum pressure of region \p RegionIdx. Also sets each
+ /// element of \p BankedPeaks to the banked VGPR peaks of the region for its
+ /// bank limits.
+ GCNRegPressure getRealRegPressure(
+ unsigned RegionIdx,
+ MutableArrayRef<GCNRegPressure::BankedVGPRPeaks> BankedPeaks = {}) const;
// Compute and cache live-ins and pressure for all regions in block.
void computeBlockPressure(unsigned RegionIdx, const MachineBasicBlock *MBB);
``````````
</details>
https://github.com/llvm/llvm-project/pull/243967
More information about the llvm-branch-commits
mailing list