[llvm] [AMDGPU] NFC: Add BBLiveOutMap & LiveOut Cache (PR #93089)
Jeffrey Byrnes via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 14 12:19:31 PDT 2024
https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/93089
>From 619c2dbf46a1817ee6ead3a80662f11f839ded06 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 21 May 2024 12:55:07 -0700
Subject: [PATCH 1/5] [AMDGPU] NFC: Add BBLiveOutMap & LiveOut Cache
Change-Id: I63cfd44e635cc4bee0e6780ca43b692c46e940b7
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 56 ++++++++++++++++++++-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 7 +++
2 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 94d93390d0916..a4d05f62a7f74 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -58,6 +58,11 @@ static cl::opt<bool>
"Wave Limited (amdgpu-limit-wave-threshold)."),
cl::init(false));
+static cl::opt<bool> GCNTrackers(
+ "amdgpu-use-amdgpu-trackers", cl::Hidden,
+ cl::desc("Use the AMDGPU specific RPTrackers during scheduling"),
+ cl::init(false));
+
const unsigned ScheduleMetrics::ScaleFactor = 100;
GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
@@ -526,6 +531,19 @@ GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const {
return RPTracker.moveMaxPressure();
}
+static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin,
+ MachineBasicBlock::iterator RegionEnd) {
+ MachineInstr *LastMI;
+ auto *BB = RegionBegin->getParent();
+ if (RegionEnd != BB->end() && !RegionEnd->isDebugInstr())
+ LastMI = &*RegionEnd;
+ else if (RegionEnd == BB->end())
+ LastMI = &*prev_nodbg(RegionEnd, RegionBegin);
+ else
+ LastMI = &*skipDebugInstructionsBackward(RegionEnd, RegionBegin);
+ return LastMI;
+}
+
void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
const MachineBasicBlock *MBB) {
GCNDownwardRPTracker RPTracker(*LIS);
@@ -597,6 +615,16 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
RPTracker.advanceBeforeNext();
MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
}
+
+ if (GCNTrackers) {
+ assert(LiveOuts.size() == Regions.size());
+ for (unsigned RegionIdx = 0; RegionIdx < Regions.size(); RegionIdx++) {
+ auto RegionBegin = Regions[RegionIdx].first;
+ auto RegionEnd = Regions[RegionIdx].second;
+ MachineInstr *LastMI = getLastMIForRegion(RegionBegin, RegionEnd);
+ LiveOuts[RegionIdx] = BBLiveOutMap.lookup(LastMI);
+ }
+ }
}
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
@@ -616,11 +644,24 @@ GCNScheduleDAGMILive::getBBLiveInMap() const {
return getLiveRegMap(BBStarters, false /*After*/, *LIS);
}
+DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
+GCNScheduleDAGMILive::getBBLiveOutMap() const {
+ assert(!Regions.empty());
+ std::vector<MachineInstr *> BBEnders;
+ BBEnders.reserve(Regions.size());
+ auto I = Regions.rbegin(), E = Regions.rend();
+ for (; I != E; I++)
+ BBEnders.push_back(getLastMIForRegion(I->first, I->second));
+
+ return getLiveRegMap(BBEnders, true /*After*/, *LIS);
+}
+
void GCNScheduleDAGMILive::finalizeSchedule() {
// Start actual scheduling here. This function is called by the base
// MachineScheduler after all regions have been recorded by
// GCNScheduleDAGMILive::schedule().
LiveIns.resize(Regions.size());
+ LiveOuts.resize(Regions.size());
Pressure.resize(Regions.size());
RescheduleRegions.resize(Regions.size());
RegionsWithHighRP.resize(Regions.size());
@@ -639,8 +680,12 @@ void GCNScheduleDAGMILive::finalizeSchedule() {
void GCNScheduleDAGMILive::runSchedStages() {
LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
- if (!Regions.empty())
+ if (!Regions.empty()) {
BBLiveInMap = getBBLiveInMap();
+ if (GCNTrackers) {
+ BBLiveOutMap = getBBLiveOutMap();
+ }
+ }
GCNSchedStrategy &S = static_cast<GCNSchedStrategy &>(*SchedImpl);
while (S.advanceStage()) {
@@ -1499,6 +1544,15 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
DAG.Regions = NewRegions;
DAG.RescheduleRegions = NewRescheduleRegions;
+ if (GCNTrackers) {
+ DAG.BBLiveOutMap = DAG.getBBLiveOutMap();
+ auto I = DAG.Regions.begin(), E = DAG.Regions.end();
+ for (; I != E; I++) {
+ MachineInstr *LastMI = getLastMIForRegion(I->first, I->second);
+ DAG.LiveOuts.push_back(DAG.BBLiveOutMap.lookup(LastMI));
+ }
+ }
+
SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
MFI.increaseOccupancy(MF, ++DAG.MinOccupancy);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2084aae4128ff..243bb7f0c094d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -205,6 +205,9 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
// Region live-in cache.
SmallVector<GCNRPTracker::LiveRegSet, 32> LiveIns;
+ // Region live-out cache.
+ SmallVector<GCNRPTracker::LiveRegSet, 32> LiveOuts;
+
// Region pressure cache.
SmallVector<GCNRegPressure, 32> Pressure;
@@ -215,6 +218,10 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getBBLiveInMap() const;
+ DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveOutMap;
+
+ DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getBBLiveOutMap() const;
+
// Return current region pressure.
GCNRegPressure getRealRegPressure(unsigned RegionIdx) const;
>From 9f2f32bb361e46e088914b804fcb635c2970c66a Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 23 May 2024 11:03:27 -0700
Subject: [PATCH 2/5] Review Comments
Change-Id: Iaeaa9bc5b037d78ab965c3bc1778d424e37eb546
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4d05f62a7f74..6f792ce24350a 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -533,15 +533,10 @@ GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const {
static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin,
MachineBasicBlock::iterator RegionEnd) {
- MachineInstr *LastMI;
- auto *BB = RegionBegin->getParent();
- if (RegionEnd != BB->end() && !RegionEnd->isDebugInstr())
- LastMI = &*RegionEnd;
- else if (RegionEnd == BB->end())
- LastMI = &*prev_nodbg(RegionEnd, RegionBegin);
- else
- LastMI = &*skipDebugInstructionsBackward(RegionEnd, RegionBegin);
- return LastMI;
+ auto REnd = RegionEnd == RegionBegin->getParent()->end()
+ ? std::prev(RegionEnd)
+ : RegionEnd;
+ return &*skipDebugInstructionsBackward(REnd, RegionBegin);
}
void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
@@ -641,7 +636,7 @@ GCNScheduleDAGMILive::getBBLiveInMap() const {
++I;
} while (I != E && I->first->getParent() == BB);
} while (I != E);
- return getLiveRegMap(BBStarters, false /*After*/, *LIS);
+ return getLiveRegMap(BBStarters, /*After=*/false, *LIS);
}
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
@@ -653,7 +648,7 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const {
for (; I != E; I++)
BBEnders.push_back(getLastMIForRegion(I->first, I->second));
- return getLiveRegMap(BBEnders, true /*After*/, *LIS);
+ return getLiveRegMap(BBEnders, /*After= */true, *LIS);
}
void GCNScheduleDAGMILive::finalizeSchedule() {
>From ea707692a8c618f7f80d815645143206a39f2dab Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 23 May 2024 11:11:50 -0700
Subject: [PATCH 3/5] Formatting
Change-Id: I8418e9dd9571feb8cdbb32623f21ecb2ff41aa9e
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 6f792ce24350a..215fe79cfc728 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -648,7 +648,7 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const {
for (; I != E; I++)
BBEnders.push_back(getLastMIForRegion(I->first, I->second));
- return getLiveRegMap(BBEnders, /*After= */true, *LIS);
+ return getLiveRegMap(BBEnders, /*After= */ true, *LIS);
}
void GCNScheduleDAGMILive::finalizeSchedule() {
>From a22d7e09fe86b2186f7aa5988f0d4f9324a6d330 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 28 May 2024 11:13:32 -0700
Subject: [PATCH 4/5] Add wrapper for BBLiveRegMap
Change-Id: I4f8fa1415194aeda4cc6318c6213e9c71a2d50dc
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 46 ++++++++++-----------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 37 ++++++++++++++---
2 files changed, 54 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 215fe79cfc728..7cd009f00a9b7 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -489,7 +489,8 @@ GCNScheduleDAGMILive::GCNScheduleDAGMILive(
MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S)
: ScheduleDAGMILive(C, std::move(S)), ST(MF.getSubtarget<GCNSubtarget>()),
MFI(*MF.getInfo<SIMachineFunctionInfo>()),
- StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy) {
+ StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy),
+ RegionLiveOuts(this, /*IsLiveOut=*/true) {
LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
if (RelaxedOcc) {
@@ -610,16 +611,6 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
RPTracker.advanceBeforeNext();
MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
}
-
- if (GCNTrackers) {
- assert(LiveOuts.size() == Regions.size());
- for (unsigned RegionIdx = 0; RegionIdx < Regions.size(); RegionIdx++) {
- auto RegionBegin = Regions[RegionIdx].first;
- auto RegionEnd = Regions[RegionIdx].second;
- MachineInstr *LastMI = getLastMIForRegion(RegionBegin, RegionEnd);
- LiveOuts[RegionIdx] = BBLiveOutMap.lookup(LastMI);
- }
- }
}
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
@@ -651,12 +642,29 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const {
return getLiveRegMap(BBEnders, /*After= */ true, *LIS);
}
+void RegionPressureMap::buildLiveRegMap() {
+ if (IsMapGenerated) {
+ IdxToInstruction.clear();
+ BBLiveRegMap.clear();
+ IsMapGenerated = false;
+ }
+
+ BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap();
+ for (unsigned I = 0; I < DAG->Regions.size(); I++) {
+ MachineInstr *RegionKey;
+ RegionKey = IsLiveOut ? getLastMIForRegion(DAG->Regions[I].first,
+ DAG->Regions[I].second)
+ : &*DAG->Regions[I].first;
+ IdxToInstruction[I] = RegionKey;
+ }
+ IsMapGenerated = true;
+}
+
void GCNScheduleDAGMILive::finalizeSchedule() {
// Start actual scheduling here. This function is called by the base
// MachineScheduler after all regions have been recorded by
// GCNScheduleDAGMILive::schedule().
LiveIns.resize(Regions.size());
- LiveOuts.resize(Regions.size());
Pressure.resize(Regions.size());
RescheduleRegions.resize(Regions.size());
RegionsWithHighRP.resize(Regions.size());
@@ -677,9 +685,8 @@ void GCNScheduleDAGMILive::runSchedStages() {
if (!Regions.empty()) {
BBLiveInMap = getBBLiveInMap();
- if (GCNTrackers) {
- BBLiveOutMap = getBBLiveOutMap();
- }
+ if (GCNTrackers)
+ RegionLiveOuts.buildLiveRegMap();
}
GCNSchedStrategy &S = static_cast<GCNSchedStrategy &>(*SchedImpl);
@@ -1539,15 +1546,6 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
DAG.Regions = NewRegions;
DAG.RescheduleRegions = NewRescheduleRegions;
- if (GCNTrackers) {
- DAG.BBLiveOutMap = DAG.getBBLiveOutMap();
- auto I = DAG.Regions.begin(), E = DAG.Regions.end();
- for (; I != E; I++) {
- MachineInstr *LastMI = getLastMIForRegion(I->first, I->second);
- DAG.LiveOuts.push_back(DAG.BBLiveOutMap.lookup(LastMI));
- }
- }
-
SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
MFI.increaseOccupancy(MF, ++DAG.MinOccupancy);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 243bb7f0c094d..9001b16ec256d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -163,6 +163,35 @@ inline raw_ostream &operator<<(raw_ostream &OS, const ScheduleMetrics &Sm) {
return OS;
}
+class GCNScheduleDAGMILive;
+class RegionPressureMap {
+ GCNScheduleDAGMILive *DAG;
+ // The live in/out pressure as indexed by the first or last MI in the region
+ // before scheduling.
+ DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveRegMap;
+ // The mapping of RegionIDx to key instruction
+ DenseMap<unsigned, MachineInstr *> IdxToInstruction;
+ // Whether we are calculating LiveOuts or LiveIns
+ bool IsLiveOut;
+ // Whether or not the maps have been generated
+ bool IsMapGenerated = false;
+
+public:
+ RegionPressureMap() {}
+ RegionPressureMap(GCNScheduleDAGMILive *GCNDAG, bool LiveOut)
+ : DAG(GCNDAG), IsLiveOut(LiveOut) {}
+ // Build the Instr->LiveReg and RegionIdx->Instr maps
+ void buildLiveRegMap();
+
+ // Retrieve the LiveReg for a given RegionIdx
+ GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) {
+ assert(IsMapGenerated);
+ assert(IdxToInstruction.find(RegionIdx) != IdxToInstruction.end());
+ MachineInstr *Key = IdxToInstruction[RegionIdx];
+ return BBLiveRegMap[Key];
+ }
+};
+
class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
friend class GCNSchedStage;
friend class OccInitialScheduleStage;
@@ -170,6 +199,7 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
friend class ClusteredLowOccStage;
friend class PreRARematStage;
friend class ILPInitialScheduleStage;
+ friend class RegionPressureMap;
const GCNSubtarget &ST;
@@ -205,9 +235,6 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
// Region live-in cache.
SmallVector<GCNRPTracker::LiveRegSet, 32> LiveIns;
- // Region live-out cache.
- SmallVector<GCNRPTracker::LiveRegSet, 32> LiveOuts;
-
// Region pressure cache.
SmallVector<GCNRegPressure, 32> Pressure;
@@ -218,10 +245,10 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getBBLiveInMap() const;
- DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveOutMap;
-
DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getBBLiveOutMap() const;
+ RegionPressureMap RegionLiveOuts;
+
// Return current region pressure.
GCNRegPressure getRealRegPressure(unsigned RegionIdx) const;
>From 18cf1b9cce0d4c8d727e525a1e43963486f98815 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 14 Jun 2024 08:21:24 -0700
Subject: [PATCH 5/5] Review comments
Change-Id: Ic703ad43fa7fd49afcb1d17c30325eb0dd7a8355
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 21 ++++++++-------------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 3 ---
2 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 7cd009f00a9b7..f770958c6c0d1 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -635,29 +635,24 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const {
assert(!Regions.empty());
std::vector<MachineInstr *> BBEnders;
BBEnders.reserve(Regions.size());
- auto I = Regions.rbegin(), E = Regions.rend();
- for (; I != E; I++)
- BBEnders.push_back(getLastMIForRegion(I->first, I->second));
+ for (auto &[RegionBegin, RegionEnd] : reverse(Regions))
+ BBEnders.push_back(getLastMIForRegion(RegionBegin, RegionEnd));
return getLiveRegMap(BBEnders, /*After= */ true, *LIS);
}
void RegionPressureMap::buildLiveRegMap() {
- if (IsMapGenerated) {
- IdxToInstruction.clear();
- BBLiveRegMap.clear();
- IsMapGenerated = false;
- }
+ IdxToInstruction.clear();
+ BBLiveRegMap.clear();
BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap();
for (unsigned I = 0; I < DAG->Regions.size(); I++) {
- MachineInstr *RegionKey;
- RegionKey = IsLiveOut ? getLastMIForRegion(DAG->Regions[I].first,
- DAG->Regions[I].second)
- : &*DAG->Regions[I].first;
+ MachineInstr *RegionKey =
+ IsLiveOut
+ ? getLastMIForRegion(DAG->Regions[I].first, DAG->Regions[I].second)
+ : &*DAG->Regions[I].first;
IdxToInstruction[I] = RegionKey;
}
- IsMapGenerated = true;
}
void GCNScheduleDAGMILive::finalizeSchedule() {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 9001b16ec256d..e7e086ffe513c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -173,8 +173,6 @@ class RegionPressureMap {
DenseMap<unsigned, MachineInstr *> IdxToInstruction;
// Whether we are calculating LiveOuts or LiveIns
bool IsLiveOut;
- // Whether or not the maps have been generated
- bool IsMapGenerated = false;
public:
RegionPressureMap() {}
@@ -185,7 +183,6 @@ class RegionPressureMap {
// Retrieve the LiveReg for a given RegionIdx
GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) {
- assert(IsMapGenerated);
assert(IdxToInstruction.find(RegionIdx) != IdxToInstruction.end());
MachineInstr *Key = IdxToInstruction[RegionIdx];
return BBLiveRegMap[Key];
More information about the llvm-commits
mailing list