[llvm] [AMDGPU][MachineScheduler] Alternative way to control excess RP. (PR #68004)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 3 09:50:02 PDT 2023
https://github.com/alex-t updated https://github.com/llvm/llvm-project/pull/68004
>From 7d894c0b039e6804079c096f1bbeb2980cafe378 Mon Sep 17 00:00:00 2001
From: Alexander Timofeev <alexander.timofeev at amd.com>
Date: Mon, 2 Oct 2023 18:35:12 +0200
Subject: [PATCH 1/2] [AMDGPU][MachineScheduler] Alternative way to control
excess RP.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 52 ++++++++++++---------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 11 +++--
2 files changed, 38 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index ce481e1f1a8bc48..793bbe90307efce 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -894,10 +894,24 @@ void GCNSchedStage::setupNewBlock() {
void GCNSchedStage::finalizeGCNRegion() {
DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd);
- DAG.RescheduleRegions[RegionIdx] = false;
- if (S.HasHighPressure)
+ PressureAfter = DAG.getRealRegPressure(RegionIdx);
+
+ unsigned NewVGPRRP = PressureAfter.getVGPRNum(false);
+ unsigned NewAGPRRP = PressureAfter.getAGPRNum();
+ unsigned NewSGPRRP = PressureAfter.getSGPRNum();
+
+ if ((NewVGPRRP >= S.VGPRCriticalLimit - S.VGPRExcessMargin) ||
+ (NewAGPRRP >= S.VGPRCriticalLimit - S.VGPRExcessMargin) ||
+ (NewSGPRRP >= S.SGPRCriticalLimit - S.SGPRExcessMargin))
DAG.RegionsWithHighRP[RegionIdx] = true;
+ if ((NewVGPRRP >= S.VGPRExcessLimit - S.VGPRExcessMargin) ||
+ (NewAGPRRP >= S.VGPRExcessLimit - S.SGPRExcessMargin) ||
+ (NewSGPRRP >= S.SGPRExcessLimit - S.VGPRExcessMargin)) {
+ DAG.RegionsWithExcessRP[RegionIdx] = true;
+ DAG.RescheduleRegions[RegionIdx] = true;
+ }
+
// Revert scheduling if we have dropped occupancy or there is some other
// reason that the original schedule is better.
checkScheduling();
@@ -912,7 +926,6 @@ void GCNSchedStage::finalizeGCNRegion() {
void GCNSchedStage::checkScheduling() {
// Check the results of scheduling.
- PressureAfter = DAG.getRealRegPressure(RegionIdx);
LLVM_DEBUG(dbgs() << "Pressure after scheduling: " << print(PressureAfter));
LLVM_DEBUG(dbgs() << "Region: " << RegionIdx << ".\n");
@@ -959,16 +972,6 @@ void GCNSchedStage::checkScheduling() {
<< DAG.MinOccupancy << ".\n");
}
- unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
- unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
- if (PressureAfter.getVGPRNum(false) > MaxVGPRs ||
- PressureAfter.getAGPRNum() > MaxVGPRs ||
- PressureAfter.getSGPRNum() > MaxSGPRs) {
- DAG.RescheduleRegions[RegionIdx] = true;
- DAG.RegionsWithHighRP[RegionIdx] = true;
- DAG.RegionsWithExcessRP[RegionIdx] = true;
- }
-
// Revert if this region's schedule would cause a drop in occupancy or
// spilling.
if (shouldRevertScheduling(WavesAfter)) {
@@ -1117,16 +1120,23 @@ bool OccInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
// If RP is not reduced in the unclustered reschedule stage, revert to the
// old schedule.
- if ((WavesAfter <= PressureBefore.getOccupancy(ST) &&
- mayCauseSpilling(WavesAfter)) ||
- GCNSchedStage::shouldRevertScheduling(WavesAfter)) {
- LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n");
- return true;
- }
+ if (DAG.RegionsWithExcessRP[RegionIdx]) {
+ unsigned NewVGPRRP = PressureAfter.getVGPRNum(false);
+ unsigned NewAGPRRP = PressureAfter.getAGPRNum();
+ unsigned NewSGPRRP = PressureAfter.getSGPRNum();
- // Do not attempt to relax schedule even more if we are already spilling.
- if (isRegionWithExcessRP())
+ unsigned OldVGPRRP = PressureBefore.getVGPRNum(false);
+ unsigned OldAGPRRP = PressureBefore.getAGPRNum();
+ unsigned OldSGPRRP = PressureBefore.getSGPRNum();
+
+ if (NewVGPRRP > S.VGPRExcessLimit && NewVGPRRP >= OldVGPRRP)
+ return true;
+ if (NewAGPRRP > S.VGPRExcessLimit && NewAGPRRP >= OldAGPRRP)
+ return true;
+ if (NewSGPRRP > S.SGPRExcessLimit && NewSGPRRP >= OldSGPRRP)
+ return true;
return false;
+ }
LLVM_DEBUG(
dbgs()
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 7862ec1e894b62e..2119a6f3109bca8 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -56,10 +56,6 @@ class GCNSchedStrategy : public GenericScheduler {
std::vector<unsigned> MaxPressure;
- unsigned SGPRExcessLimit;
-
- unsigned VGPRExcessLimit;
-
unsigned TargetOccupancy;
MachineFunction *MF;
@@ -94,10 +90,17 @@ class GCNSchedStrategy : public GenericScheduler {
unsigned VGPRCriticalLimit;
+ unsigned SGPRExcessLimit;
+
+ unsigned VGPRExcessLimit;
+
unsigned SGPRLimitBias = 0;
unsigned VGPRLimitBias = 0;
+ unsigned VGPRExcessMargin = 1;
+ unsigned SGPRExcessMargin = 0;
+
GCNSchedStrategy(const MachineSchedContext *C);
SUnit *pickNode(bool &IsTopNode) override;
>From f262452050fd5a5a5da2d3ae4245650eccddb89d Mon Sep 17 00:00:00 2001
From: Alexander Timofeev <alexander.timofeev at amd.com>
Date: Mon, 2 Oct 2023 18:35:12 +0200
Subject: [PATCH 2/2] [AMDGPU][MachineScheduler] Alternative way to control
excess RP.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 50 ++++++++++++---------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 11 +++--
2 files changed, 36 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index ce481e1f1a8bc48..e26ad2ccd705328 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -702,7 +702,7 @@ bool UnclusteredHighRPStage::initGCNSchedStage() {
if (!GCNSchedStage::initGCNSchedStage())
return false;
- if (DAG.RegionsWithHighRP.none() && DAG.RegionsWithExcessRP.none())
+ if (DAG.RegionsWithExcessRP.none())
return false;
SavedMutations.swap(DAG.Mutations);
@@ -894,10 +894,22 @@ void GCNSchedStage::setupNewBlock() {
void GCNSchedStage::finalizeGCNRegion() {
DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd);
- DAG.RescheduleRegions[RegionIdx] = false;
+ PressureAfter = DAG.getRealRegPressure(RegionIdx);
+
if (S.HasHighPressure)
DAG.RegionsWithHighRP[RegionIdx] = true;
+ unsigned NewVGPRRP = PressureAfter.getVGPRNum(false);
+ unsigned NewAGPRRP = PressureAfter.getAGPRNum();
+ unsigned NewSGPRRP = PressureAfter.getSGPRNum();
+
+ if ((NewVGPRRP >= S.VGPRExcessLimit - S.VGPRExcessMargin) ||
+ (NewAGPRRP >= S.VGPRExcessLimit - S.SGPRExcessMargin) ||
+ (NewSGPRRP >= S.SGPRExcessLimit - S.VGPRExcessMargin)) {
+ DAG.RegionsWithExcessRP[RegionIdx] = true;
+ DAG.RescheduleRegions[RegionIdx] = true;
+ }
+
// Revert scheduling if we have dropped occupancy or there is some other
// reason that the original schedule is better.
checkScheduling();
@@ -912,7 +924,6 @@ void GCNSchedStage::finalizeGCNRegion() {
void GCNSchedStage::checkScheduling() {
// Check the results of scheduling.
- PressureAfter = DAG.getRealRegPressure(RegionIdx);
LLVM_DEBUG(dbgs() << "Pressure after scheduling: " << print(PressureAfter));
LLVM_DEBUG(dbgs() << "Region: " << RegionIdx << ".\n");
@@ -959,16 +970,6 @@ void GCNSchedStage::checkScheduling() {
<< DAG.MinOccupancy << ".\n");
}
- unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
- unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
- if (PressureAfter.getVGPRNum(false) > MaxVGPRs ||
- PressureAfter.getAGPRNum() > MaxVGPRs ||
- PressureAfter.getSGPRNum() > MaxSGPRs) {
- DAG.RescheduleRegions[RegionIdx] = true;
- DAG.RegionsWithHighRP[RegionIdx] = true;
- DAG.RegionsWithExcessRP[RegionIdx] = true;
- }
-
// Revert if this region's schedule would cause a drop in occupancy or
// spilling.
if (shouldRevertScheduling(WavesAfter)) {
@@ -1117,16 +1118,23 @@ bool OccInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
// If RP is not reduced in the unclustered reschedule stage, revert to the
// old schedule.
- if ((WavesAfter <= PressureBefore.getOccupancy(ST) &&
- mayCauseSpilling(WavesAfter)) ||
- GCNSchedStage::shouldRevertScheduling(WavesAfter)) {
- LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n");
- return true;
- }
+ if (DAG.RegionsWithExcessRP[RegionIdx]) {
+ unsigned NewVGPRRP = PressureAfter.getVGPRNum(false);
+ unsigned NewAGPRRP = PressureAfter.getAGPRNum();
+ unsigned NewSGPRRP = PressureAfter.getSGPRNum();
- // Do not attempt to relax schedule even more if we are already spilling.
- if (isRegionWithExcessRP())
+ unsigned OldVGPRRP = PressureBefore.getVGPRNum(false);
+ unsigned OldAGPRRP = PressureBefore.getAGPRNum();
+ unsigned OldSGPRRP = PressureBefore.getSGPRNum();
+
+ if (NewVGPRRP > S.VGPRExcessLimit && NewVGPRRP >= OldVGPRRP)
+ return true;
+ if (NewAGPRRP > S.VGPRExcessLimit && NewAGPRRP >= OldAGPRRP)
+ return true;
+ if (NewSGPRRP > S.SGPRExcessLimit && NewSGPRRP >= OldSGPRRP)
+ return true;
return false;
+ }
LLVM_DEBUG(
dbgs()
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 7862ec1e894b62e..2119a6f3109bca8 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -56,10 +56,6 @@ class GCNSchedStrategy : public GenericScheduler {
std::vector<unsigned> MaxPressure;
- unsigned SGPRExcessLimit;
-
- unsigned VGPRExcessLimit;
-
unsigned TargetOccupancy;
MachineFunction *MF;
@@ -94,10 +90,17 @@ class GCNSchedStrategy : public GenericScheduler {
unsigned VGPRCriticalLimit;
+ unsigned SGPRExcessLimit;
+
+ unsigned VGPRExcessLimit;
+
unsigned SGPRLimitBias = 0;
unsigned VGPRLimitBias = 0;
+ unsigned VGPRExcessMargin = 1;
+ unsigned SGPRExcessMargin = 0;
+
GCNSchedStrategy(const MachineSchedContext *C);
SUnit *pickNode(bool &IsTopNode) override;
More information about the llvm-commits
mailing list