[llvm] r303179 - [AMDGPU] Turn register pressure estimation into forward tracker
Stanislav Mekhanoshin via llvm-commits
llvm-commits at lists.llvm.org
Tue May 16 08:43:52 PDT 2017
Author: rampitec
Date: Tue May 16 10:43:52 2017
New Revision: 303179
URL: http://llvm.org/viewvc/llvm-project?rev=303179&view=rev
Log:
[AMDGPU] Turn register pressure estimation into forward tracker
This factors register pressure estimation mechanism from the
GCNSchedStrategy into the forward tracker to unify interface
with other strategies and expose it to other interested phases.
Differential Revision: https://reviews.llvm.org/D33105
Modified:
llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp
llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h
llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.cpp
llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.h
Modified: llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp?rev=303179&r1=303178&r2=303179&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp Tue May 16 10:43:52 2017
@@ -27,7 +27,7 @@ void llvm::printLivesAt(SlotIndex SI,
unsigned Num = 0;
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
const unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
- if (MRI.reg_nodbg_empty(Reg))
+ if (!LIS.hasInterval(Reg))
continue;
const auto &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
@@ -192,7 +192,6 @@ LaneBitmask llvm::getLiveLaneMask(unsign
SlotIndex SI,
const LiveIntervals &LIS,
const MachineRegisterInfo &MRI) {
- assert(!MRI.reg_nodbg_empty(Reg));
LaneBitmask LiveMask;
const auto &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
@@ -214,7 +213,7 @@ GCNRPTracker::LiveRegSet llvm::getLiveRe
GCNRPTracker::LiveRegSet LiveRegs;
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
auto Reg = TargetRegisterInfo::index2VirtReg(I);
- if (MRI.reg_nodbg_empty(Reg))
+ if (!LIS.hasInterval(Reg))
continue;
auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
if (LiveMask.any())
@@ -223,13 +222,7 @@ GCNRPTracker::LiveRegSet llvm::getLiveRe
return LiveRegs;
}
-void GCNUpwardRPTracker::reset(const MachineInstr &MI) {
- MRI = &MI.getParent()->getParent()->getRegInfo();
- LiveRegs = getLiveRegsAfter(MI, LIS);
- MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
-}
-
-LaneBitmask GCNUpwardRPTracker::getDefRegMask(const MachineOperand &MO) const {
+LaneBitmask GCNRPTracker::getDefRegMask(const MachineOperand &MO) const {
assert(MO.isDef() && MO.isReg() &&
TargetRegisterInfo::isVirtualRegister(MO.getReg()));
@@ -241,7 +234,7 @@ LaneBitmask GCNUpwardRPTracker::getDefRe
MRI->getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
}
-LaneBitmask GCNUpwardRPTracker::getUsedRegMask(const MachineOperand &MO) const {
+LaneBitmask GCNRPTracker::getUsedRegMask(const MachineOperand &MO) const {
assert(MO.isUse() && MO.isReg() &&
TargetRegisterInfo::isVirtualRegister(MO.getReg()));
@@ -259,6 +252,18 @@ LaneBitmask GCNUpwardRPTracker::getUsedR
return getLiveLaneMask(MO.getReg(), SI, LIS, *MRI);
}
+void GCNUpwardRPTracker::reset(const MachineInstr &MI,
+ const LiveRegSet *LiveRegsCopy) {
+ MRI = &MI.getParent()->getParent()->getRegInfo();
+ if (LiveRegsCopy) {
+ if (&LiveRegs != LiveRegsCopy)
+ LiveRegs = *LiveRegsCopy;
+ } else {
+ LiveRegs = getLiveRegsAfter(MI, LIS);
+ }
+ MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
+}
+
void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
assert(MRI && "call reset first");
@@ -297,6 +302,100 @@ void GCNUpwardRPTracker::recede(const Ma
MaxPressure = max(MaxPressure, CurPressure);
}
+bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
+ const LiveRegSet *LiveRegsCopy) {
+ MRI = &MI.getParent()->getParent()->getRegInfo();
+ LastTrackedMI = nullptr;
+ MBBEnd = MI.getParent()->end();
+ NextMI = &MI;
+ NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ if (NextMI == MBBEnd)
+ return false;
+ if (LiveRegsCopy) {
+ if (&LiveRegs != LiveRegsCopy)
+ LiveRegs = *LiveRegsCopy;
+ } else {
+ LiveRegs = getLiveRegsBefore(*NextMI, LIS);
+ }
+ MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
+ return true;
+}
+
+bool GCNDownwardRPTracker::advanceBeforeNext() {
+ assert(MRI && "call reset first");
+
+ NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ if (NextMI == MBBEnd)
+ return false;
+
+ SlotIndex SI = LIS.getInstructionIndex(*NextMI).getBaseIndex();
+ assert(SI.isValid());
+
+ // Remove dead registers or mask bits.
+ for (auto &It : LiveRegs) {
+ const LiveInterval &LI = LIS.getInterval(It.first);
+ if (LI.hasSubRanges()) {
+ for (const auto &S : LI.subranges()) {
+ if (!S.liveAt(SI)) {
+ auto PrevMask = It.second;
+ It.second &= ~S.LaneMask;
+ CurPressure.inc(It.first, PrevMask, It.second, *MRI);
+ }
+ }
+ } else if (!LI.liveAt(SI)) {
+ auto PrevMask = It.second;
+ It.second = LaneBitmask::getNone();
+ CurPressure.inc(It.first, PrevMask, It.second, *MRI);
+ }
+ if (It.second.none())
+ LiveRegs.erase(It.first);
+ }
+
+ MaxPressure = max(MaxPressure, CurPressure);
+
+ return true;
+}
+
+void GCNDownwardRPTracker::advanceToNext() {
+ LastTrackedMI = &*NextMI++;
+
+ // Add new registers or mask bits.
+ for (const auto &MO : LastTrackedMI->defs()) {
+ if (!MO.isReg())
+ continue;
+ unsigned Reg = MO.getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(Reg))
+ continue;
+ auto &LiveMask = LiveRegs[Reg];
+ auto PrevMask = LiveMask;
+ LiveMask |= getDefRegMask(MO);
+ CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ }
+
+ MaxPressure = max(MaxPressure, CurPressure);
+}
+
+bool GCNDownwardRPTracker::advance() {
+ // If we have just called reset live set is actual.
+ if ((NextMI == MBBEnd) || (LastTrackedMI && !advanceBeforeNext()))
+ return false;
+ advanceToNext();
+ return true;
+}
+
+bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
+ while (NextMI != End)
+ if (!advance()) return false;
+ return true;
+}
+
+bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
+ MachineBasicBlock::const_iterator End,
+ const LiveRegSet *LiveRegsCopy) {
+ reset(*Begin, LiveRegsCopy);
+ return advance(End);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD
static void reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
Modified: llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h?rev=303179&r1=303178&r2=303179&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h Tue May 16 10:43:52 2017
@@ -92,16 +92,21 @@ public:
typedef DenseMap<unsigned, LaneBitmask> LiveRegSet;
protected:
+ const LiveIntervals &LIS;
LiveRegSet LiveRegs;
GCNRegPressure CurPressure, MaxPressure;
const MachineInstr *LastTrackedMI = nullptr;
mutable const MachineRegisterInfo *MRI = nullptr;
- GCNRPTracker() {}
+ GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
+ LaneBitmask getDefRegMask(const MachineOperand &MO) const;
+ LaneBitmask getUsedRegMask(const MachineOperand &MO) const;
public:
// live regs for the current state
const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
+ void clearMaxPressure() { MaxPressure.clear(); }
+
// returns MaxPressure, resetting it
decltype(MaxPressure) moveMaxPressure() {
auto Res = MaxPressure;
@@ -114,14 +119,11 @@ public:
};
class GCNUpwardRPTracker : public GCNRPTracker {
- const LiveIntervals &LIS;
- LaneBitmask getDefRegMask(const MachineOperand &MO) const;
- LaneBitmask getUsedRegMask(const MachineOperand &MO) const;
public:
- GCNUpwardRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
+ GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
// reset tracker to the point just below MI
// filling live regs upon this point using LIS
- void reset(const MachineInstr &MI);
+ void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
// move to the state just above the MI
void recede(const MachineInstr &MI);
@@ -131,6 +133,41 @@ public:
bool isValid() const;
};
+class GCNDownwardRPTracker : public GCNRPTracker {
+ // Last position of reset or advanceBeforeNext
+ MachineBasicBlock::const_iterator NextMI;
+
+ MachineBasicBlock::const_iterator MBBEnd;
+
+public:
+ GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
+
+ const MachineBasicBlock::const_iterator getNext() const { return NextMI; }
+
+ // Reset tracker to the point before the MI
+ // filling live regs upon this point using LIS.
+ // Returns false if block is empty except debug values.
+ bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
+
+ // Move to the state right before the next MI. Returns false if reached
+ // end of the block.
+ bool advanceBeforeNext();
+
+ // Move to the state at the MI, advanceBeforeNext has to be called first.
+ void advanceToNext();
+
+ // Move to the state at the next MI. Returns false if reached end of block.
+ bool advance();
+
+ // Advance instructions until before End.
+ bool advance(MachineBasicBlock::const_iterator End);
+
+ // Reset to Begin and advance to End.
+ bool advance(MachineBasicBlock::const_iterator Begin,
+ MachineBasicBlock::const_iterator End,
+ const LiveRegSet *LiveRegsCopy = nullptr);
+};
+
LaneBitmask getLiveLaneMask(unsigned Reg,
SlotIndex SI,
const LiveIntervals &LIS,
Modified: llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.cpp?rev=303179&r1=303178&r2=303179&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.cpp Tue May 16 10:43:52 2017
@@ -327,11 +327,15 @@ void GCNScheduleDAGMILive::schedule() {
for (auto &I : *this)
Unsched.push_back(&I);
- std::pair<unsigned, unsigned> PressureBefore;
+ GCNRegPressure PressureBefore;
if (LIS) {
- DEBUG(dbgs() << "Pressure before scheduling:\n");
discoverLiveIns();
PressureBefore = getRealRegPressure();
+
+ DEBUG(dbgs() << "Pressure before scheduling:\nSGPR = "
+ << PressureBefore.getSGPRNum()
+ << "\nVGPR = " << PressureBefore.getVGPRNum() << '\n');
+
}
ScheduleDAGMILive::schedule();
@@ -343,19 +347,23 @@ void GCNScheduleDAGMILive::schedule() {
// Check the results of scheduling.
GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
- DEBUG(dbgs() << "Pressure after scheduling:\n");
auto PressureAfter = getRealRegPressure();
+
+ DEBUG(dbgs() << "Pressure after scheduling:\nSGPR = "
+ << PressureAfter.getSGPRNum()
+ << "\nVGPR = " << PressureAfter.getVGPRNum() << '\n');
+
LiveIns.clear();
- if (PressureAfter.first <= S.SGPRCriticalLimit &&
- PressureAfter.second <= S.VGPRCriticalLimit) {
+ if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
+ PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
DEBUG(dbgs() << "Pressure in desired limits, done.\n");
return;
}
- unsigned WavesAfter = getMaxWaves(PressureAfter.first,
- PressureAfter.second, MF);
- unsigned WavesBefore = getMaxWaves(PressureBefore.first,
- PressureBefore.second, MF);
+ unsigned WavesAfter = getMaxWaves(PressureAfter.getSGPRNum(),
+ PressureAfter.getVGPRNum(), MF);
+ unsigned WavesBefore = getMaxWaves(PressureBefore.getSGPRNum(),
+ PressureBefore.getVGPRNum(), MF);
DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore <<
", after " << WavesAfter << ".\n");
@@ -404,112 +412,31 @@ void GCNScheduleDAGMILive::schedule() {
placeDebugValues();
}
-static inline void setMask(const MachineRegisterInfo &MRI,
- const SIRegisterInfo *SRI, unsigned Reg,
- LaneBitmask &PrevMask, LaneBitmask NewMask,
- unsigned &SGPRs, unsigned &VGPRs) {
- int NewRegs = countPopulation(NewMask.getAsInteger()) -
- countPopulation(PrevMask.getAsInteger());
- if (SRI->isSGPRReg(MRI, Reg))
- SGPRs += NewRegs;
- if (SRI->isVGPR(MRI, Reg))
- VGPRs += NewRegs;
- assert ((int)SGPRs >= 0 && (int)VGPRs >= 0);
- PrevMask = NewMask;
-}
-
void GCNScheduleDAGMILive::discoverLiveIns() {
- unsigned SGPRs = 0;
- unsigned VGPRs = 0;
-
- auto I = begin();
- I = skipDebugInstructionsForward(I, I->getParent()->end());
- const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
- SlotIndex SI = LIS->getInstructionIndex(*I).getBaseIndex();
- assert (SI.isValid());
-
- DEBUG(dbgs() << "Region live-ins:");
- for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
- unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
- if (MRI.reg_nodbg_empty(Reg))
- continue;
- const LiveInterval &LI = LIS->getInterval(Reg);
- LaneBitmask LaneMask = LaneBitmask::getNone();
- if (LI.hasSubRanges()) {
- for (const auto &S : LI.subranges())
- if (S.liveAt(SI))
- LaneMask |= S.LaneMask;
- } else if (LI.liveAt(SI)) {
- LaneMask = MRI.getMaxLaneMaskForVReg(Reg);
- }
-
- if (LaneMask.any()) {
- setMask(MRI, SRI, Reg, LiveIns[Reg], LaneMask, SGPRs, VGPRs);
+ GCNDownwardRPTracker RPTracker(*LIS);
+ RPTracker.reset(*begin());
- DEBUG(dbgs() << ' ' << PrintVRegOrUnit(Reg, SRI) << ':'
- << PrintLaneMask(LiveIns[Reg]));
- }
- }
-
- LiveInPressure = std::make_pair(SGPRs, VGPRs);
+ LiveIns = RPTracker.moveLiveRegs();
- DEBUG(dbgs() << "\nLive-in pressure:\nSGPR = " << SGPRs
- << "\nVGPR = " << VGPRs << '\n');
+ DEBUG(GCNRegPressure LiveInPressure = RPTracker.moveMaxPressure();
+ const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
+ dbgs() << "Region live-ins:";
+ for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
+ unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
+ auto It = LiveIns.find(Reg);
+ if (It != LiveIns.end())
+ dbgs() << ' ' << PrintVRegOrUnit(Reg, SRI) << ':'
+ << PrintLaneMask(It->second);
+ }
+ dbgs() << "\nLive-in pressure:\nSGPR = "
+ << LiveInPressure.getSGPRNum()
+ << "\nVGPR = " << LiveInPressure.getVGPRNum() << '\n');
}
-std::pair<unsigned, unsigned>
-GCNScheduleDAGMILive::getRealRegPressure() const {
- unsigned SGPRs, MaxSGPRs, VGPRs, MaxVGPRs;
- SGPRs = MaxSGPRs = LiveInPressure.first;
- VGPRs = MaxVGPRs = LiveInPressure.second;
-
- const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
- DenseMap<unsigned, LaneBitmask> LiveRegs(LiveIns);
-
- for (const MachineInstr &MI : *this) {
- if (MI.isDebugValue())
- continue;
- SlotIndex SI = LIS->getInstructionIndex(MI).getBaseIndex();
- assert (SI.isValid());
-
- // Remove dead registers or mask bits.
- for (auto &It : LiveRegs) {
- if (It.second.none())
- continue;
- const LiveInterval &LI = LIS->getInterval(It.first);
- if (LI.hasSubRanges()) {
- for (const auto &S : LI.subranges())
- if (!S.liveAt(SI))
- setMask(MRI, SRI, It.first, It.second, It.second & ~S.LaneMask,
- SGPRs, VGPRs);
- } else if (!LI.liveAt(SI)) {
- setMask(MRI, SRI, It.first, It.second, LaneBitmask::getNone(),
- SGPRs, VGPRs);
- }
- }
-
- // Add new registers or mask bits.
- for (const auto &MO : MI.defs()) {
- if (!MO.isReg())
- continue;
- unsigned Reg = MO.getReg();
- if (!TargetRegisterInfo::isVirtualRegister(Reg))
- continue;
- unsigned SubRegIdx = MO.getSubReg();
- LaneBitmask LaneMask = SubRegIdx != 0
- ? TRI->getSubRegIndexLaneMask(SubRegIdx)
- : MRI.getMaxLaneMaskForVReg(Reg);
- LaneBitmask &LM = LiveRegs[Reg];
- setMask(MRI, SRI, Reg, LM, LM | LaneMask, SGPRs, VGPRs);
- }
- MaxSGPRs = std::max(MaxSGPRs, SGPRs);
- MaxVGPRs = std::max(MaxVGPRs, VGPRs);
- }
-
- DEBUG(dbgs() << "Real region's register pressure:\nSGPR = " << MaxSGPRs
- << "\nVGPR = " << MaxVGPRs << '\n');
-
- return std::make_pair(MaxSGPRs, MaxVGPRs);
+GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
+ GCNDownwardRPTracker RPTracker(*LIS);
+ RPTracker.advance(begin(), end(), &LiveIns);
+ return RPTracker.moveMaxPressure();
}
void GCNScheduleDAGMILive::finalizeSchedule() {
Modified: llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.h?rev=303179&r1=303178&r2=303179&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNSchedStrategy.h Tue May 16 10:43:52 2017
@@ -14,6 +14,7 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
#define LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
+#include "GCNRegPressure.h"
#include "llvm/CodeGen/MachineScheduler.h"
namespace llvm {
@@ -79,16 +80,13 @@ class GCNScheduleDAGMILive : public Sche
MachineBasicBlock::iterator>, 32> Regions;
// Region live-ins.
- DenseMap<unsigned, LaneBitmask> LiveIns;
-
- // Number of live-ins to the current region, first SGPR then VGPR.
- std::pair<unsigned, unsigned> LiveInPressure;
+ GCNRPTracker::LiveRegSet LiveIns;
// Collect current region live-ins.
void discoverLiveIns();
- // Return current region pressure. First value is SGPR number, second is VGPR.
- std::pair<unsigned, unsigned> getRealRegPressure() const;
+ // Return current region pressure.
+ GCNRegPressure getRealRegPressure() const;
public:
GCNScheduleDAGMILive(MachineSchedContext *C,
More information about the llvm-commits
mailing list