[llvm] [AMDGPU] NFC: Provide RPTracker interface for external iterators (PR #93088)
Jeffrey Byrnes via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 21 15:39:19 PDT 2024
https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/93088
>From 8aa83aa035f027d4cdc6211ec945d961abd72c47 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 21 May 2024 13:34:59 -0700
Subject: [PATCH 01/19] [AMDGPU] NFC: Provide RPTracker interface for external
iterators
Change-Id: I79b54722e6e858961486248d94766c3f3c161160
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 70 +++++++++++++++--------
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 18 +++---
2 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 5c394e6d6296d0..f1c4c8b397ddc6 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -343,24 +343,25 @@ void GCNRPTracker::reset(const MachineInstr &MI,
MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
}
-////////////////////////////////////////////////////////////////////////////////
-// GCNUpwardRPTracker
-
-void GCNUpwardRPTracker::reset(const MachineRegisterInfo &MRI_,
- const LiveRegSet &LiveRegs_) {
+void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
+ const LiveRegSet &LiveRegs_) {
MRI = &MRI_;
LiveRegs = LiveRegs_;
LastTrackedMI = nullptr;
MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);
}
-void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
+////////////////////////////////////////////////////////////////////////////////
+// GCNUpwardRPTracker
+
+bool GCNUpwardRPTracker::recede(const MachineInstr &MI, bool ShouldTrackIt) {
assert(MRI && "call reset first");
- LastTrackedMI = &MI;
+ if (ShouldTrackIt)
+ LastTrackedMI = &MI;
if (MI.isDebugInstr())
- return;
+ return false;
// Kill all defs.
GCNRegPressure DefPressure, ECDefPressure;
@@ -412,6 +413,7 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
: max(CurPressure, MaxPressure);
assert(CurPressure == getRegPressure(*MRI, LiveRegs));
+ return false;
}
////////////////////////////////////////////////////////////////////////////////
@@ -430,28 +432,44 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
return true;
}
-bool GCNDownwardRPTracker::advanceBeforeNext() {
+bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
+ bool ShouldTrackIt,
+ LiveIntervals *TheLIS) {
assert(MRI && "call reset first");
- if (!LastTrackedMI)
- return NextMI == MBBEnd;
+ SlotIndex SI;
+ LiveIntervals *CurrLIS;
+ MachineInstr *CurrMI;
+ if (ShouldTrackIt) {
+ if (!LastTrackedMI)
+ return NextMI == MBBEnd;
+
+ assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
+ CurrLIS = const_cast<LiveIntervals *>(&LIS);
+ CurrMI = const_cast<MachineInstr *>(LastTrackedMI);
+
+ SI = NextMI == MBBEnd
+ ? CurrLIS->getInstructionIndex(*LastTrackedMI).getDeadSlot()
+ : CurrLIS->getInstructionIndex(*NextMI).getBaseIndex();
+ }
- assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
+ else { //! ShouldTrackIt
+ CurrLIS = TheLIS;
+ SI = CurrLIS->getInstructionIndex(*MI).getBaseIndex();
+ CurrMI = MI;
+ }
- SlotIndex SI = NextMI == MBBEnd
- ? LIS.getInstructionIndex(*LastTrackedMI).getDeadSlot()
- : LIS.getInstructionIndex(*NextMI).getBaseIndex();
assert(SI.isValid());
// Remove dead registers or mask bits.
SmallSet<Register, 8> SeenRegs;
- for (auto &MO : LastTrackedMI->operands()) {
+ for (auto &MO : CurrMI->operands()) {
if (!MO.isReg() || !MO.getReg().isVirtual())
continue;
if (MO.isUse() && !MO.readsReg())
continue;
if (!SeenRegs.insert(MO.getReg()).second)
continue;
- const LiveInterval &LI = LIS.getInterval(MO.getReg());
+ const LiveInterval &LI = CurrLIS->getInterval(MO.getReg());
if (LI.hasSubRanges()) {
auto It = LiveRegs.end();
for (const auto &S : LI.subranges()) {
@@ -481,15 +499,18 @@ bool GCNDownwardRPTracker::advanceBeforeNext() {
LastTrackedMI = nullptr;
- return NextMI == MBBEnd;
+ return ShouldTrackIt && (NextMI == MBBEnd);
}
-void GCNDownwardRPTracker::advanceToNext() {
+void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI, bool ShouldTrackIt) {
LastTrackedMI = &*NextMI++;
NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ MachineInstr *CurrMI =
+ ShouldTrackIt ? const_cast<MachineInstr *>(LastTrackedMI) : MI;
+
// Add new registers or mask bits.
- for (const auto &MO : LastTrackedMI->all_defs()) {
+ for (const auto &MO : CurrMI->all_defs()) {
Register Reg = MO.getReg();
if (!Reg.isVirtual())
continue;
@@ -502,11 +523,12 @@ void GCNDownwardRPTracker::advanceToNext() {
MaxPressure = max(MaxPressure, CurPressure);
}
-bool GCNDownwardRPTracker::advance() {
- if (NextMI == MBBEnd)
+bool GCNDownwardRPTracker::advance(MachineInstr *MI, bool ShouldTrackIt,
+ LiveIntervals *TheLIS) {
+ if (ShouldTrackIt && NextMI == MBBEnd)
return false;
- advanceBeforeNext();
- advanceToNext();
+ advanceBeforeNext(MI, ShouldTrackIt, TheLIS);
+ advanceToNext(MI, ShouldTrackIt);
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 752f53752fa68b..8abbce138cf164 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -160,6 +160,9 @@ class GCNRPTracker {
bool After);
public:
+ // reset tracker and set live register set to the specified value.
+ void reset(const MachineRegisterInfo &MRI_, const LiveRegSet &LiveRegs_);
+
// live regs for the current state
const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
@@ -180,12 +183,9 @@ class GCNUpwardRPTracker : public GCNRPTracker {
public:
GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
- // reset tracker and set live register set to the specified value.
- void reset(const MachineRegisterInfo &MRI_, const LiveRegSet &LiveRegs_);
-
// reset tracker at the specified slot index.
void reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
- reset(MRI, llvm::getLiveRegs(SI, LIS, MRI));
+ GCNRPTracker::reset(MRI, llvm::getLiveRegs(SI, LIS, MRI));
}
// reset tracker to the end of the MBB.
@@ -200,7 +200,7 @@ class GCNUpwardRPTracker : public GCNRPTracker {
}
// move to the state just before the MI (in program order).
- void recede(const MachineInstr &MI);
+ bool recede(const MachineInstr &MI, bool ShouldTrackIt = true);
// checks whether the tracker's state after receding MI corresponds
// to reported by LIS.
@@ -242,13 +242,15 @@ class GCNDownwardRPTracker : public GCNRPTracker {
// Move to the state right before the next MI or after the end of MBB.
// Returns false if reached end of the block.
- bool advanceBeforeNext();
+ bool advanceBeforeNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
+ LiveIntervals *TheLIS = nullptr);
// Move to the state at the MI, advanceBeforeNext has to be called first.
- void advanceToNext();
+ void advanceToNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true);
// Move to the state at the next MI. Returns false if reached end of block.
- bool advance();
+ bool advance(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
+ LiveIntervals *TheLIS = nullptr);
// Advance instructions until before End.
bool advance(MachineBasicBlock::const_iterator End);
>From cd91b8f3f6ef6a0a56e4f6fb0a5bfde6c8959c7f Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 23 May 2024 11:11:38 -0700
Subject: [PATCH 02/19] Review comments
Change-Id: Ib798a9d6add7d6dd1ccd0e01bea09ac2f4eeb94f
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 15 +++---
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 64 +++++++++++++++++------
2 files changed, 54 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index f1c4c8b397ddc6..6957bf8da6713a 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -354,14 +354,14 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
////////////////////////////////////////////////////////////////////////////////
// GCNUpwardRPTracker
-bool GCNUpwardRPTracker::recede(const MachineInstr &MI, bool ShouldTrackIt) {
+void GCNUpwardRPTracker::recede(const MachineInstr &MI, bool ShouldTrackIt) {
assert(MRI && "call reset first");
if (ShouldTrackIt)
LastTrackedMI = &MI;
if (MI.isDebugInstr())
- return false;
+ return;
// Kill all defs.
GCNRegPressure DefPressure, ECDefPressure;
@@ -413,7 +413,6 @@ bool GCNUpwardRPTracker::recede(const MachineInstr &MI, bool ShouldTrackIt) {
: max(CurPressure, MaxPressure);
assert(CurPressure == getRegPressure(*MRI, LiveRegs));
- return false;
}
////////////////////////////////////////////////////////////////////////////////
@@ -450,9 +449,7 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
SI = NextMI == MBBEnd
? CurrLIS->getInstructionIndex(*LastTrackedMI).getDeadSlot()
: CurrLIS->getInstructionIndex(*NextMI).getBaseIndex();
- }
-
- else { //! ShouldTrackIt
+ } else { //! ShouldTrackIt
CurrLIS = TheLIS;
SI = CurrLIS->getInstructionIndex(*MI).getBaseIndex();
CurrMI = MI;
@@ -503,8 +500,10 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
}
void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI, bool ShouldTrackIt) {
- LastTrackedMI = &*NextMI++;
- NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ if (ShouldTrackIt) {
+ LastTrackedMI = &*NextMI++;
+ NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ }
MachineInstr *CurrMI =
ShouldTrackIt ? const_cast<MachineInstr *>(LastTrackedMI) : MI;
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 8abbce138cf164..54e320cf08707e 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -143,6 +143,9 @@ inline GCNRegPressure operator-(const GCNRegPressure &P1,
return Diff;
}
+///////////////////////////////////////////////////////////////////////////////
+// GCNRPTracker
+
class GCNRPTracker {
public:
using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
@@ -179,31 +182,36 @@ class GCNRPTracker {
GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, const LiveIntervals &LIS,
const MachineRegisterInfo &MRI);
+////////////////////////////////////////////////////////////////////////////////
+// GCNUpwardRPTracker
+
class GCNUpwardRPTracker : public GCNRPTracker {
public:
GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
- // reset tracker at the specified slot index.
+ /// reset tracker at the specified slot index \p SI.
void reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
GCNRPTracker::reset(MRI, llvm::getLiveRegs(SI, LIS, MRI));
}
- // reset tracker to the end of the MBB.
+ /// reset tracker to the end of the \p MBB.
void reset(const MachineBasicBlock &MBB) {
reset(MBB.getParent()->getRegInfo(),
LIS.getSlotIndexes()->getMBBEndIdx(&MBB));
}
- // reset tracker to the point just after MI (in program order).
+ /// reset tracker to the point just after \p MI (in program order).
void reset(const MachineInstr &MI) {
reset(MI.getMF()->getRegInfo(), LIS.getInstructionIndex(MI).getDeadSlot());
}
- // move to the state just before the MI (in program order).
- bool recede(const MachineInstr &MI, bool ShouldTrackIt = true);
+ /// Move to the state of RP just before the \p MI . If \p ShouldTrackIt is
+ /// set, also update the internal iterators. Setting \p ShouldTrackIt to false
+ /// allows for an externally managed iterator / program order.
+ void recede(const MachineInstr &MI, bool ShouldTrackIt = true);
- // checks whether the tracker's state after receding MI corresponds
- // to reported by LIS.
+ /// \p returns whether the tracker's state after receding MI corresponds
+ /// to reported by LIS.
bool isValid() const;
const GCNRegPressure &getMaxPressure() const { return MaxPressure; }
@@ -217,6 +225,9 @@ class GCNUpwardRPTracker : public GCNRPTracker {
}
};
+////////////////////////////////////////////////////////////////////////////////
+// GCNDownwardRPTracker
+
class GCNDownwardRPTracker : public GCNRPTracker {
// Last position of reset or advanceBeforeNext
MachineBasicBlock::const_iterator NextMI;
@@ -228,34 +239,53 @@ class GCNDownwardRPTracker : public GCNRPTracker {
MachineBasicBlock::const_iterator getNext() const { return NextMI; }
- // Return MaxPressure and clear it.
+ /// \p return MaxPressure and clear it.
GCNRegPressure moveMaxPressure() {
auto Res = MaxPressure;
MaxPressure.clear();
return Res;
}
- // Reset tracker to the point before the MI
- // filling live regs upon this point using LIS.
- // Returns false if block is empty except debug values.
+ /// Reset tracker to the point before the \p MI
+ /// filling \p LiveRegs upon this point using LIS.
+ /// \p returns false if block is empty except debug values.
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
- // Move to the state right before the next MI or after the end of MBB.
- // Returns false if reached end of the block.
+ /// Move to the state right before the next MI or after the end of MBB.
+ /// \p returns false if reached end of the block.
+ /// If \p ShouldTrackIt is true, then internal iterators are used and set to
+ /// process in program order.
+ /// If \p ShouldTrackIt is false, then it is assumed that the tracker is using
+ /// an externally managed iterator, and advance* calls will not update the
+ /// state of the iterator. In such cases, the tracker will move to the state
+ /// right before the provided \p MI and use the provided \p TheLIS for RP
+ /// calculations.
bool advanceBeforeNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
LiveIntervals *TheLIS = nullptr);
- // Move to the state at the MI, advanceBeforeNext has to be called first.
+ /// Move to the state at the MI, advanceBeforeNext has to be called first.
+ /// If \p ShouldTrackIt is true, then internal iterators are used and set to
+ /// process in program order.
+ /// If \p ShouldTrackIt is false, then it is assumed that the tracker is using
+ /// an externally managed iterator, and advance* calls will not update the
+ /// state of the iterator. In such cases, the tracker will move to the state
+ /// at the provided \p MI .
void advanceToNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true);
- // Move to the state at the next MI. Returns false if reached end of block.
+ /// Move to the state at the next MI. \p returns false if reached end of
+ /// block. If \p ShouldTrackIt is true, then internal iterators are used and
+ /// set to process in program order. If \p ShouldTrackIt is false, then it is
+ /// assumed that the tracker is using an externally managed iterator, and
+ /// advance* calls will not update the state of the iterator. In such cases,
+ /// the tracker will move to the state right before the provided \p MI and use
+ /// the provided \p TheLIS for RP calculations.
bool advance(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
LiveIntervals *TheLIS = nullptr);
- // Advance instructions until before End.
+ /// Advance instructions until before \p End.
bool advance(MachineBasicBlock::const_iterator End);
- // Reset to Begin and advance to End.
+ /// Reset to \p Begin and advance to \p End.
bool advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
const LiveRegSet *LiveRegsCopy = nullptr);
>From 6a6026d0aba8ef8f4e9ec9ed330873832f10ca73 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 27 May 2024 10:40:10 -0700
Subject: [PATCH 03/19] Fix RP calculation behavior
Change-Id: I10242186f538359ff09110dd70b23e5136655849
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 6957bf8da6713a..e420ed06be6b09 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -464,6 +464,8 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
continue;
if (MO.isUse() && !MO.readsReg())
continue;
+ if (!ShouldTrackIt && MO.isDef())
+ continue;
if (!SeenRegs.insert(MO.getReg()).second)
continue;
const LiveInterval &LI = CurrLIS->getInterval(MO.getReg());
>From 5ad16447ff1667859ba02bf3febab72d03e09e50 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 14 Jun 2024 13:32:56 -0700
Subject: [PATCH 04/19] Review comments
Change-Id: I5c25a2e899c46fc3026b215010bde32b6c6d24ac
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 30 ++++++------
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 56 ++++++++++++-----------
2 files changed, 46 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index e420ed06be6b09..a2d76807d3a716 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -354,10 +354,11 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
////////////////////////////////////////////////////////////////////////////////
// GCNUpwardRPTracker
-void GCNUpwardRPTracker::recede(const MachineInstr &MI, bool ShouldTrackIt) {
+void GCNUpwardRPTracker::recede(const MachineInstr &MI,
+ bool UseInternalIterator) {
assert(MRI && "call reset first");
- if (ShouldTrackIt)
+ if (UseInternalIterator)
LastTrackedMI = &MI;
if (MI.isDebugInstr())
@@ -432,13 +433,13 @@ bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
}
bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
- bool ShouldTrackIt,
+ bool UseInternalIterator,
LiveIntervals *TheLIS) {
assert(MRI && "call reset first");
SlotIndex SI;
LiveIntervals *CurrLIS;
MachineInstr *CurrMI;
- if (ShouldTrackIt) {
+ if (UseInternalIterator) {
if (!LastTrackedMI)
return NextMI == MBBEnd;
@@ -449,7 +450,7 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
SI = NextMI == MBBEnd
? CurrLIS->getInstructionIndex(*LastTrackedMI).getDeadSlot()
: CurrLIS->getInstructionIndex(*NextMI).getBaseIndex();
- } else { //! ShouldTrackIt
+ } else { //! UseInternalIterator
CurrLIS = TheLIS;
SI = CurrLIS->getInstructionIndex(*MI).getBaseIndex();
CurrMI = MI;
@@ -464,7 +465,7 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
continue;
if (MO.isUse() && !MO.readsReg())
continue;
- if (!ShouldTrackIt && MO.isDef())
+ if (!UseInternalIterator && MO.isDef())
continue;
if (!SeenRegs.insert(MO.getReg()).second)
continue;
@@ -498,17 +499,18 @@ bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
LastTrackedMI = nullptr;
- return ShouldTrackIt && (NextMI == MBBEnd);
+ return UseInternalIterator && (NextMI == MBBEnd);
}
-void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI, bool ShouldTrackIt) {
- if (ShouldTrackIt) {
+void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,
+ bool UseInternalIterator) {
+ if (UseInternalIterator) {
LastTrackedMI = &*NextMI++;
NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
}
MachineInstr *CurrMI =
- ShouldTrackIt ? const_cast<MachineInstr *>(LastTrackedMI) : MI;
+ UseInternalIterator ? const_cast<MachineInstr *>(LastTrackedMI) : MI;
// Add new registers or mask bits.
for (const auto &MO : CurrMI->all_defs()) {
@@ -524,12 +526,12 @@ void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI, bool ShouldTrackIt) {
MaxPressure = max(MaxPressure, CurPressure);
}
-bool GCNDownwardRPTracker::advance(MachineInstr *MI, bool ShouldTrackIt,
+bool GCNDownwardRPTracker::advance(MachineInstr *MI, bool UseInternalIterator,
LiveIntervals *TheLIS) {
- if (ShouldTrackIt && NextMI == MBBEnd)
+ if (UseInternalIterator && NextMI == MBBEnd)
return false;
- advanceBeforeNext(MI, ShouldTrackIt, TheLIS);
- advanceToNext(MI, ShouldTrackIt);
+ advanceBeforeNext(MI, UseInternalIterator, TheLIS);
+ advanceToNext(MI, UseInternalIterator);
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 54e320cf08707e..6ae20dad8e9a30 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -189,6 +189,8 @@ class GCNUpwardRPTracker : public GCNRPTracker {
public:
GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
+ using GCNRPTracker::reset;
+
/// reset tracker at the specified slot index \p SI.
void reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
GCNRPTracker::reset(MRI, llvm::getLiveRegs(SI, LIS, MRI));
@@ -205,10 +207,10 @@ class GCNUpwardRPTracker : public GCNRPTracker {
reset(MI.getMF()->getRegInfo(), LIS.getInstructionIndex(MI).getDeadSlot());
}
- /// Move to the state of RP just before the \p MI . If \p ShouldTrackIt is
- /// set, also update the internal iterators. Setting \p ShouldTrackIt to false
- /// allows for an externally managed iterator / program order.
- void recede(const MachineInstr &MI, bool ShouldTrackIt = true);
+ /// Move to the state of RP just before the \p MI . If \p UseInternalIterator
+ /// is set, also update the internal iterators. Setting \p UseInternalIterator
+ /// to false allows for an externally managed iterator / program order.
+ void recede(const MachineInstr &MI, bool UseInternalIterator = true);
/// \p returns whether the tracker's state after receding MI corresponds
/// to reported by LIS.
@@ -237,6 +239,8 @@ class GCNDownwardRPTracker : public GCNRPTracker {
public:
GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
+ using GCNRPTracker::reset;
+
MachineBasicBlock::const_iterator getNext() const { return NextMI; }
/// \p return MaxPressure and clear it.
@@ -253,33 +257,33 @@ class GCNDownwardRPTracker : public GCNRPTracker {
/// Move to the state right before the next MI or after the end of MBB.
/// \p returns false if reached end of the block.
- /// If \p ShouldTrackIt is true, then internal iterators are used and set to
- /// process in program order.
- /// If \p ShouldTrackIt is false, then it is assumed that the tracker is using
- /// an externally managed iterator, and advance* calls will not update the
- /// state of the iterator. In such cases, the tracker will move to the state
- /// right before the provided \p MI and use the provided \p TheLIS for RP
- /// calculations.
- bool advanceBeforeNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
+ /// If \p UseInternalIterator is true, then internal iterators are used and
+ /// set to process in program order. If \p UseInternalIterator is false, then
+ /// it is assumed that the tracker is using an externally managed iterator,
+ /// and advance* calls will not update the state of the iterator. In such
+ /// cases, the tracker will move to the state right before the provided \p MI
+ /// and use the provided \p TheLIS for RP calculations.
+ bool advanceBeforeNext(MachineInstr *MI = nullptr,
+ bool UseInternalIterator = true,
LiveIntervals *TheLIS = nullptr);
/// Move to the state at the MI, advanceBeforeNext has to be called first.
- /// If \p ShouldTrackIt is true, then internal iterators are used and set to
- /// process in program order.
- /// If \p ShouldTrackIt is false, then it is assumed that the tracker is using
- /// an externally managed iterator, and advance* calls will not update the
- /// state of the iterator. In such cases, the tracker will move to the state
- /// at the provided \p MI .
- void advanceToNext(MachineInstr *MI = nullptr, bool ShouldTrackIt = true);
+ /// If \p UseInternalIterator is true, then internal iterators are used and
+ /// set to process in program order. If \p UseInternalIterator is false, then
+ /// it is assumed that the tracker is using an externally managed iterator,
+ /// and advance* calls will not update the state of the iterator. In such
+ /// cases, the tracker will move to the state at the provided \p MI .
+ void advanceToNext(MachineInstr *MI = nullptr,
+ bool UseInternalIterator = true);
/// Move to the state at the next MI. \p returns false if reached end of
- /// block. If \p ShouldTrackIt is true, then internal iterators are used and
- /// set to process in program order. If \p ShouldTrackIt is false, then it is
- /// assumed that the tracker is using an externally managed iterator, and
- /// advance* calls will not update the state of the iterator. In such cases,
- /// the tracker will move to the state right before the provided \p MI and use
- /// the provided \p TheLIS for RP calculations.
- bool advance(MachineInstr *MI = nullptr, bool ShouldTrackIt = true,
+ /// block. If \p UseInternalIterator is true, then internal iterators are used
+ /// and set to process in program order. If \p UseInternalIterator is false,
+ /// then it is assumed that the tracker is using an externally managed
+ /// iterator, and advance* calls will not update the state of the iterator. In
+ /// such cases, the tracker will move to the state right before the provided
+ /// \p MI and use the provided \p TheLIS for RP calculations.
+ bool advance(MachineInstr *MI = nullptr, bool UseInternalIterator = true,
LiveIntervals *TheLIS = nullptr);
/// Advance instructions until before \p End.
>From 4ed7a1d004cd24cfc60eb8b4da72db92e962dfa0 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 18 Jun 2024 11:48:55 -0700
Subject: [PATCH 05/19] Remove flag from upward RPTracker
Change-Id: I9633e160b23360da06520b63a0e9b7c0fd33647a
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 6 ++----
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 2 +-
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index a2d76807d3a716..dd2da917308d5d 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -354,12 +354,10 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
////////////////////////////////////////////////////////////////////////////////
// GCNUpwardRPTracker
-void GCNUpwardRPTracker::recede(const MachineInstr &MI,
- bool UseInternalIterator) {
+void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
assert(MRI && "call reset first");
- if (UseInternalIterator)
- LastTrackedMI = &MI;
+ LastTrackedMI = &MI;
if (MI.isDebugInstr())
return;
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 6ae20dad8e9a30..cfb18bf999a289 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -210,7 +210,7 @@ class GCNUpwardRPTracker : public GCNRPTracker {
/// Move to the state of RP just before the \p MI . If \p UseInternalIterator
/// is set, also update the internal iterators. Setting \p UseInternalIterator
/// to false allows for an externally managed iterator / program order.
- void recede(const MachineInstr &MI, bool UseInternalIterator = true);
+ void recede(const MachineInstr &MI);
/// \p returns whether the tracker's state after receding MI corresponds
/// to reported by LIS.
>From d7418d8ffac373ada0831990c15feb8e5d0ae834 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 11:56:05 -0700
Subject: [PATCH 06/19] Port bumpDeadDefs
Change-Id: I3b23fd089a1350570162633a4318feca092e472a
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 15 +++++++++++++++
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 3 +++
2 files changed, 18 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index dd2da917308d5d..f44799bea2b983 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -351,6 +351,21 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);
}
+void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
+ for (const RegisterMaskPair &P : DeadDefs) {
+ Register Reg = P.RegUnit;
+ LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask BumpedMask = LiveMask | P.LaneMask;
+ increaseRegPressure(Reg, LiveMask, BumpedMask);
+ }
+ for (const RegisterMaskPair &P : DeadDefs) {
+ Register Reg = P.RegUnit;
+ LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask BumpedMask = LiveMask | P.LaneMask;
+ decreaseRegPressure(Reg, BumpedMask, LiveMask);
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
// GCNUpwardRPTracker
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index cfb18bf999a289..55ab68b467892b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -19,6 +19,7 @@
#include "GCNSubtarget.h"
#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/RegisterPressure.h"
#include <algorithm>
namespace llvm {
@@ -161,6 +162,8 @@ class GCNRPTracker {
void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,
bool After);
+
+ void bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs);
public:
// reset tracker and set live register set to the specified value.
>From 3491af16b8121dc7413fe2f291527d1917035bcd Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 8 Aug 2024 11:17:34 -0700
Subject: [PATCH 07/19] Fix bumpDeadDefs port issues
Change-Id: I1d557aa00cfb57f7a0395131ef57ac19efb5d5cf
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index f44799bea2b983..f206a0eac2bf05 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -354,15 +354,16 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
for (const RegisterMaskPair &P : DeadDefs) {
Register Reg = P.RegUnit;
- LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask BumpedMask = LiveMask | P.LaneMask;
- increaseRegPressure(Reg, LiveMask, BumpedMask);
+ CurPressure.inc(Reg, LiveMask, BumpedMask, *MRI);
}
+ MaxPressure = max(MaxPressure, CurPressure);
for (const RegisterMaskPair &P : DeadDefs) {
Register Reg = P.RegUnit;
- LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask BumpedMask = LiveMask | P.LaneMask;
- decreaseRegPressure(Reg, BumpedMask, LiveMask);
+ CurPressure.inc(Reg, BumpedMask, LiveMask, *MRI);
}
}
>From 0f6cf8c1043e76a2706f6bc3e0891c2b4179c454 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 10:27:02 -0700
Subject: [PATCH 08/19] Port getRegLanes
Change-Id: I1cc390ff9a10fac9e874b097e3279376e52209a3
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index f206a0eac2bf05..8f42046c0312b0 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -288,6 +288,16 @@ collectVirtualRegUses(SmallVectorImpl<RegisterMaskPair> &RegMaskPairs,
}
}
+static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
+ Register RegUnit) {
+ auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
+ return Other.RegUnit == RegUnit;
+ });
+ if (I == RegUnits.end())
+ return LaneBitmask::getNone();
+ return I->LaneMask;
+}
+
///////////////////////////////////////////////////////////////////////////////
// GCNRPTracker
>From f8fb08b0b5b87d1ae2e5fdcdbe55e000a020f38a Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 10:30:28 -0700
Subject: [PATCH 09/19] Port getLanesWithProperty
Change-Id: Idba7806fd4d6d43ba0381d7731720ac358c108e6
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 29 +++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 8f42046c0312b0..839694357dfbbc 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -298,6 +298,35 @@ static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
return I->LaneMask;
}
+static LaneBitmask
+getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
+ bool TrackLaneMasks, Register RegUnit, SlotIndex Pos,
+ LaneBitmask SafeDefault,
+ bool (*Property)(const LiveRange &LR, SlotIndex Pos)) {
+ if (RegUnit.isVirtual()) {
+ const LiveInterval &LI = LIS.getInterval(RegUnit);
+ LaneBitmask Result;
+ if (TrackLaneMasks && LI.hasSubRanges()) {
+ for (const LiveInterval::SubRange &SR : LI.subranges()) {
+ if (Property(SR, Pos))
+ Result |= SR.LaneMask;
+ }
+ } else if (Property(LI, Pos)) {
+ Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit)
+ : LaneBitmask::getAll();
+ }
+
+ return Result;
+ } else {
+ const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
+ // Be prepared for missing liveranges: We usually do not compute liveranges
+ // for physical registers on targets with many registers (GPUs).
+ if (LR == nullptr)
+ return SafeDefault;
+ return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
+ }
+}
+
///////////////////////////////////////////////////////////////////////////////
// GCNRPTracker
>From a07dc633ad4f78a474e8f82250cffcc066166e0f Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 10:43:36 -0700
Subject: [PATCH 10/19] Port getLastUsedLanes
Change-Id: I233f0d3b3ce89adbc5a5c22efa2f28d5fb438e4a
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 11 +++++++++++
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 2 ++
2 files changed, 13 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 839694357dfbbc..180f765e54a94e 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -406,6 +406,17 @@ void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
}
}
+LaneBitmask GCNRPTracker::getLastUsedLanes(Register RegUnit,
+ SlotIndex Pos) const {
+ assert(RequireIntervals);
+ return getLanesWithProperty(
+ *LIS, *MRI, TrackLaneMasks, RegUnit, Pos.getBaseIndex(),
+ LaneBitmask::getNone(), [](const LiveRange &LR, SlotIndex Pos) {
+ const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
+ return S != nullptr && S->end == Pos.getRegSlot();
+ });
+}
+
////////////////////////////////////////////////////////////////////////////////
// GCNUpwardRPTracker
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 55ab68b467892b..b5690ed77cd6b5 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -165,6 +165,8 @@ class GCNRPTracker {
void bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs);
+ LaneBitmask getLastUsedLanes(Register RegUnit, SlotIndex Pos) const;
+
public:
// reset tracker and set live register set to the specified value.
void reset(const MachineRegisterInfo &MRI_, const LiveRegSet &LiveRegs_);
>From 1d2bb5bb7d06730be9e94ca51789e95ae4357b36 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 10:48:21 -0700
Subject: [PATCH 11/19] Fix port issues
Change-Id: I78f62d348b90448fa934aee1e9900634bf8dc209
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 180f765e54a94e..dc329aead581d7 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -408,10 +408,9 @@ void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
LaneBitmask GCNRPTracker::getLastUsedLanes(Register RegUnit,
SlotIndex Pos) const {
- assert(RequireIntervals);
return getLanesWithProperty(
- *LIS, *MRI, TrackLaneMasks, RegUnit, Pos.getBaseIndex(),
- LaneBitmask::getNone(), [](const LiveRange &LR, SlotIndex Pos) {
+ LIS, *MRI, true, RegUnit, Pos.getBaseIndex(), LaneBitmask::getNone(),
+ [](const LiveRange &LR, SlotIndex Pos) {
const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
return S != nullptr && S->end == Pos.getRegSlot();
});
>From 042f1e95b2a03c82ab42a35cdef59652f71da0a0 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 12:13:39 -0700
Subject: [PATCH 12/19] Port bumpUpwardPressure
Change-Id: Ie030633023361be626a34a1867e7740777a575d9
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 47 +++++++++++++++++++++++
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 2 +
2 files changed, 49 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index dc329aead581d7..7b61931acc2641 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -479,6 +479,53 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
assert(CurPressure == getRegPressure(*MRI, LiveRegs));
}
+void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
+ assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
+
+ SlotIndex SlotIdx;
+ if (RequireIntervals)
+ SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
+
+ // Account for register pressure similar to RegPressureTracker::recede().
+ RegisterOperands RegOpers;
+ RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);
+ assert(RegOpers.DeadDefs.empty());
+ if (TrackLaneMasks)
+ RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
+ else if (RequireIntervals)
+ RegOpers.detectDeadDefs(*MI, *LIS);
+
+ // Boost max pressure for all dead defs together.
+ // Since CurrSetPressure and MaxSetPressure
+ bumpDeadDefs(RegOpers.DeadDefs);
+
+ // Kill liveness at live defs.
+ for (const RegisterMaskPair &P : RegOpers.Defs) {
+ Register Reg = P.RegUnit;
+ LaneBitmask LiveAfter = LiveRegs.contains(Reg);
+ LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
+ LaneBitmask DefLanes = P.LaneMask;
+ LaneBitmask LiveBefore = (LiveAfter & ~DefLanes) | UseLanes;
+
+ // There may be parts of the register that were dead before the
+ // instruction, but became live afterwards. Similarly, some parts
+ // may have been killed in this instruction.
+ decreaseRegPressure(Reg, LiveAfter, LiveAfter & LiveBefore);
+ increaseRegPressure(Reg, LiveAfter, ~LiveAfter & LiveBefore);
+ }
+ // Generate liveness for uses.
+ for (const RegisterMaskPair &P : RegOpers.Uses) {
+ Register Reg = P.RegUnit;
+ // If this register was also in a def operand, we've handled it
+ // with defs.
+ if (getRegLanes(RegOpers.Defs, Reg).any())
+ continue;
+ LaneBitmask LiveAfter = LiveRegs.contains(Reg);
+ LaneBitmask LiveBefore = LiveAfter | P.LaneMask;
+ increaseRegPressure(Reg, LiveAfter, LiveBefore);
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
// GCNDownwardRPTracker
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index b5690ed77cd6b5..e5f5e46d2c6bd3 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -217,6 +217,8 @@ class GCNUpwardRPTracker : public GCNRPTracker {
/// to false allows for an externally managed iterator / program order.
void recede(const MachineInstr &MI);
+ void bumpUpwardPressure(const MachineInstr *MI);
+
/// \p returns whether the tracker's state after receding MI corresponds
/// to reported by LIS.
bool isValid() const;
>From a7b22622eab869ea5f9488cca4ccd2b8f73a0aab Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 8 Aug 2024 11:30:10 -0700
Subject: [PATCH 13/19] Fix bumpUpwardPressure port issues
hange-Id: Ib383230c9fc88fb59a4d3c44f4b1459050554aa5
Change-Id: I0333ac7f2000e9c8fccb369e330882530ca0c912
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 25 +++++++++++------------
1 file changed, 12 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 7b61931acc2641..e44ef22fb9576b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -482,18 +482,15 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
- SlotIndex SlotIdx;
- if (RequireIntervals)
- SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
+ SlotIndex SlotIdx = LIS.getInstructionIndex(*MI).getRegSlot();
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);
+ const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
+ RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/true);
assert(RegOpers.DeadDefs.empty());
- if (TrackLaneMasks)
- RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
- else if (RequireIntervals)
- RegOpers.detectDeadDefs(*MI, *LIS);
+ RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
+ RegOpers.detectDeadDefs(*MI, LIS);
// Boost max pressure for all dead defs together.
// Since CurrSetPressure and MaxSetPressure
@@ -502,7 +499,7 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// Kill liveness at live defs.
for (const RegisterMaskPair &P : RegOpers.Defs) {
Register Reg = P.RegUnit;
- LaneBitmask LiveAfter = LiveRegs.contains(Reg);
+ LaneBitmask LiveAfter = LiveRegs[Reg];
LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
LaneBitmask DefLanes = P.LaneMask;
LaneBitmask LiveBefore = (LiveAfter & ~DefLanes) | UseLanes;
@@ -510,8 +507,9 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// There may be parts of the register that were dead before the
// instruction, but became live afterwards. Similarly, some parts
// may have been killed in this instruction.
- decreaseRegPressure(Reg, LiveAfter, LiveAfter & LiveBefore);
- increaseRegPressure(Reg, LiveAfter, ~LiveAfter & LiveBefore);
+ CurPressure.inc(Reg, LiveAfter, LiveAfter & LiveBefore, *MRI);
+ CurPressure.inc(Reg, LiveAfter, ~LiveAfter & LiveBefore, *MRI);
+ MaxPressure = max(MaxPressure, CurPressure);
}
// Generate liveness for uses.
for (const RegisterMaskPair &P : RegOpers.Uses) {
@@ -520,10 +518,11 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// with defs.
if (getRegLanes(RegOpers.Defs, Reg).any())
continue;
- LaneBitmask LiveAfter = LiveRegs.contains(Reg);
+ LaneBitmask LiveAfter = LiveRegs[Reg];
LaneBitmask LiveBefore = LiveAfter | P.LaneMask;
- increaseRegPressure(Reg, LiveAfter, LiveBefore);
+ CurPressure.inc(Reg, LiveAfter, LiveBefore, *MRI);
}
+ MaxPressure = max(MaxPressure, CurPressure);
}
////////////////////////////////////////////////////////////////////////////////
>From 0e5da56e0fde51f8ee3bfe7e542fdda43118fe36 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 12:30:03 -0700
Subject: [PATCH 14/19] Port findUseBetween
Change-Id: I523e493fc80247497fd53925f05b9a8ad5f3771a
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 24 +++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index e44ef22fb9576b..d0ab133245e28c 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -327,6 +327,30 @@ getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
}
}
+/// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
+/// The query starts with a lane bitmask which gets lanes/bits removed for every
+/// use we find.
+static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
+ SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
+ const MachineRegisterInfo &MRI,
+ const LiveIntervals *LIS) {
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+ for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
+ if (MO.isUndef())
+ continue;
+ const MachineInstr *MI = MO.getParent();
+ SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
+ if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) {
+ unsigned SubRegIdx = MO.getSubReg();
+ LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
+ LastUseMask &= ~UseMask;
+ if (LastUseMask.none())
+ return LaneBitmask::getNone();
+ }
+ }
+ return LastUseMask;
+}
+
///////////////////////////////////////////////////////////////////////////////
// GCNRPTracker
>From 87ca0712b5cf3ad7aa59795b7fe4acca4692d58c Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 12:23:29 -0700
Subject: [PATCH 15/19] Port bumpDownwardPressure
Change-Id: Ib207ef130a024b3fd12f4c34e80f59dd23450ed2
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 48 +++++++++++++++++++++++
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 2 +
2 files changed, 50 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index d0ab133245e28c..86b90d08d7337a 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -706,6 +706,54 @@ Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
});
}
+void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
+ assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
+
+ SlotIndex SlotIdx;
+ if (RequireIntervals)
+ SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
+
+ // Account for register pressure similar to RegPressureTracker::recede().
+ RegisterOperands RegOpers;
+ RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/false);
+ if (TrackLaneMasks)
+ RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
+
+ if (RequireIntervals) {
+ for (const RegisterMaskPair &Use : RegOpers.Uses) {
+ Register Reg = Use.RegUnit;
+ LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
+ if (LastUseMask.none())
+ continue;
+ // The LastUseMask is queried from the liveness information of instruction
+ // which may be further down the schedule. Some lanes may actually not be
+ // last uses for the current position.
+ // FIXME: allow the caller to pass in the list of vreg uses that remain
+ // to be bottom-scheduled to avoid searching uses at each query.
+ SlotIndex CurrIdx = getCurrSlot();
+ LastUseMask =
+ findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
+ if (LastUseMask.none())
+ continue;
+
+ LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask NewMask = LiveMask & ~LastUseMask;
+ decreaseRegPressure(Reg, LiveMask, NewMask);
+ }
+ }
+
+ // Generate liveness for defs.
+ for (const RegisterMaskPair &Def : RegOpers.Defs) {
+ Register Reg = Def.RegUnit;
+ LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask NewMask = LiveMask | Def.LaneMask;
+ increaseRegPressure(Reg, LiveMask, NewMask);
+ }
+
+ // Boost pressure for all dead defs together.
+ bumpDeadDefs(RegOpers.DeadDefs);
+}
+
bool GCNUpwardRPTracker::isValid() const {
const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();
const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index e5f5e46d2c6bd3..29ebeb1843c56c 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -300,6 +300,8 @@ class GCNDownwardRPTracker : public GCNRPTracker {
bool advance(MachineBasicBlock::const_iterator Begin,
MachineBasicBlock::const_iterator End,
const LiveRegSet *LiveRegsCopy = nullptr);
+
+ void bumpDownwardPressure(const MachineInstr *MI);
};
LaneBitmask getLiveLaneMask(unsigned Reg,
>From 9d37355b0f3d79a489e45eb1dcfaa3d1cbc7914c Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 12:25:06 -0700
Subject: [PATCH 16/19] Fix bumpDownwardPressure port issues
Change-Id: I2e3c08b4487262cb290cd070f7c2042156217fb1
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 61 +++++++++++++----------
1 file changed, 34 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 86b90d08d7337a..9d87d9e7d734eb 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -710,45 +710,52 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
SlotIndex SlotIdx;
- if (RequireIntervals)
- SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
+ SlotIdx = LIS.getInstructionIndex(*MI).getRegSlot();
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/false);
- if (TrackLaneMasks)
- RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
-
- if (RequireIntervals) {
- for (const RegisterMaskPair &Use : RegOpers.Uses) {
- Register Reg = Use.RegUnit;
- LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
- if (LastUseMask.none())
- continue;
- // The LastUseMask is queried from the liveness information of instruction
- // which may be further down the schedule. Some lanes may actually not be
- // last uses for the current position.
- // FIXME: allow the caller to pass in the list of vreg uses that remain
- // to be bottom-scheduled to avoid searching uses at each query.
- SlotIndex CurrIdx = getCurrSlot();
- LastUseMask =
- findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
- if (LastUseMask.none())
- continue;
+ const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
+ RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/false);
+ RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
- LaneBitmask LiveMask = LiveRegs.contains(Reg);
- LaneBitmask NewMask = LiveMask & ~LastUseMask;
- decreaseRegPressure(Reg, LiveMask, NewMask);
+ for (const RegisterMaskPair &Use : RegOpers.Uses) {
+ Register Reg = Use.RegUnit;
+ LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
+ if (LastUseMask.none())
+ continue;
+ // The LastUseMask is queried from the liveness information of instruction
+ // which may be further down the schedule. Some lanes may actually not be
+ // last uses for the current position.
+ // FIXME: allow the caller to pass in the list of vreg uses that remain
+ // to be bottom-scheduled to avoid searching uses at each query.
+ SlotIndex CurrIdx;
+ const MachineBasicBlock *MBB = MI->getParent();
+ MachineBasicBlock::const_iterator IdxPos = skipDebugInstructionsForward(
+ LastTrackedMI ? LastTrackedMI : MBB->begin(), MBB->end());
+ if (IdxPos == MBB->end()) {
+ CurrIdx = LIS.getMBBEndIdx(MBB);
+ } else {
+ CurrIdx = LIS.getInstructionIndex(*IdxPos).getRegSlot();
}
+
+ LastUseMask =
+ findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, &LIS);
+ if (LastUseMask.none())
+ continue;
+
+ LaneBitmask LiveMask = LiveRegs[Reg];
+ LaneBitmask NewMask = LiveMask & ~LastUseMask;
+ CurPressure.inc(Reg, LiveMask, NewMask, *MRI);
}
// Generate liveness for defs.
for (const RegisterMaskPair &Def : RegOpers.Defs) {
Register Reg = Def.RegUnit;
- LaneBitmask LiveMask = LiveRegs.contains(Reg);
+ LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask NewMask = LiveMask | Def.LaneMask;
- increaseRegPressure(Reg, LiveMask, NewMask);
+ CurPressure.inc(Reg, LiveMask, NewMask, *MRI);
}
+ MaxPressure = max(MaxPressure, CurPressure);
// Boost pressure for all dead defs together.
bumpDeadDefs(RegOpers.DeadDefs);
>From 13439a4e4dcfebb9ddeb218fd47fb18465f01a50 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 12 Aug 2024 12:25:06 -0700
Subject: [PATCH 17/19] Modify behavior to support RP subreg aware RP
speculation
Change-Id: I2e3c08b4487262cb290cd070f7c2042156217fb1
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 33 +++++++++++++++++++----
1 file changed, 28 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 9d87d9e7d734eb..0cf699e9ec8722 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -333,14 +333,17 @@ getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
const MachineRegisterInfo &MRI,
- const LiveIntervals *LIS) {
+ const LiveIntervals *LIS,
+ bool Upward = false) {
const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
if (MO.isUndef())
continue;
const MachineInstr *MI = MO.getParent();
SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
- if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) {
+ bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)
+ : (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx);
+ if (InRange) {
unsigned SubRegIdx = MO.getSubReg();
LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
LastUseMask &= ~UseMask;
@@ -417,6 +420,8 @@ void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,
void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
for (const RegisterMaskPair &P : DeadDefs) {
Register Reg = P.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask BumpedMask = LiveMask | P.LaneMask;
CurPressure.inc(Reg, LiveMask, BumpedMask, *MRI);
@@ -424,6 +429,8 @@ void GCNRPTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
MaxPressure = max(MaxPressure, CurPressure);
for (const RegisterMaskPair &P : DeadDefs) {
Register Reg = P.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask BumpedMask = LiveMask | P.LaneMask;
CurPressure.inc(Reg, BumpedMask, LiveMask, *MRI);
@@ -523,6 +530,8 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// Kill liveness at live defs.
for (const RegisterMaskPair &P : RegOpers.Defs) {
Register Reg = P.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
LaneBitmask LiveAfter = LiveRegs[Reg];
LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
LaneBitmask DefLanes = P.LaneMask;
@@ -538,12 +547,21 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// Generate liveness for uses.
for (const RegisterMaskPair &P : RegOpers.Uses) {
Register Reg = P.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
// If this register was also in a def operand, we've handled it
// with defs.
if (getRegLanes(RegOpers.Defs, Reg).any())
continue;
LaneBitmask LiveAfter = LiveRegs[Reg];
- LaneBitmask LiveBefore = LiveAfter | P.LaneMask;
+ SlotIndex CurrIdx =
+ LastTrackedMI ? LIS.getInstructionIndex(*LastTrackedMI).getRegSlot()
+ : LIS.getMBBEndIdx(MI->getParent());
+ ;
+ LaneBitmask LastUseMask =
+ findUseBetween(Reg, P.LaneMask, SlotIdx, CurrIdx, *MRI, &LIS, true);
+ LastUseMask &= ~LiveAfter;
+ LaneBitmask LiveBefore = (LiveAfter | LastUseMask);
CurPressure.inc(Reg, LiveAfter, LiveBefore, *MRI);
}
MaxPressure = max(MaxPressure, CurPressure);
@@ -640,10 +658,11 @@ void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,
if (UseInternalIterator) {
LastTrackedMI = &*NextMI++;
NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
+ } else {
+ LastTrackedMI = MI;
}
- MachineInstr *CurrMI =
- UseInternalIterator ? const_cast<MachineInstr *>(LastTrackedMI) : MI;
+ MachineInstr *CurrMI = const_cast<MachineInstr *>(LastTrackedMI);
// Add new registers or mask bits.
for (const auto &MO : CurrMI->all_defs()) {
@@ -720,6 +739,8 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
for (const RegisterMaskPair &Use : RegOpers.Uses) {
Register Reg = Use.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
if (LastUseMask.none())
continue;
@@ -751,6 +772,8 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
// Generate liveness for defs.
for (const RegisterMaskPair &Def : RegOpers.Defs) {
Register Reg = Def.RegUnit;
+ if (!Reg.isVirtual())
+ continue;
LaneBitmask LiveMask = LiveRegs[Reg];
LaneBitmask NewMask = LiveMask | Def.LaneMask;
CurPressure.inc(Reg, LiveMask, NewMask, *MRI);
>From fe1bf2b37abccc4882d15ac0e6b5a1db28691f3e Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 20 Aug 2024 12:29:33 -0700
Subject: [PATCH 18/19] Review comments
Change-Id: I2de464b32d3c6ed9a77cbbc669d735dde63c2e47
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 45 +++++++++++++----------
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 0cf699e9ec8722..4374719645aec4 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -298,11 +298,11 @@ static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
return I->LaneMask;
}
-static LaneBitmask
-getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
- bool TrackLaneMasks, Register RegUnit, SlotIndex Pos,
- LaneBitmask SafeDefault,
- bool (*Property)(const LiveRange &LR, SlotIndex Pos)) {
+static LaneBitmask getLanesWithProperty(
+ const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
+ bool TrackLaneMasks, Register RegUnit, SlotIndex Pos,
+ LaneBitmask SafeDefault,
+ function_ref<bool(const LiveRange &LR, SlotIndex Pos)> Property) {
if (RegUnit.isVirtual()) {
const LiveInterval &LI = LIS.getInterval(RegUnit);
LaneBitmask Result;
@@ -317,14 +317,14 @@ getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
}
return Result;
- } else {
- const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
- // Be prepared for missing liveranges: We usually do not compute liveranges
- // for physical registers on targets with many registers (GPUs).
- if (LR == nullptr)
- return SafeDefault;
- return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
}
+
+ const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
+ // Be prepared for missing liveranges: We usually do not compute liveranges
+ // for physical registers on targets with many registers (GPUs).
+ if (LR == nullptr)
+ return SafeDefault;
+ return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
}
/// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
@@ -333,19 +333,21 @@ getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
const MachineRegisterInfo &MRI,
+ const SIRegisterInfo *TRI,
const LiveIntervals *LIS,
bool Upward = false) {
- const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
- for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
+ for (const MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {
if (MO.isUndef())
continue;
+ if (!MO.readsReg())
+ continue;
const MachineInstr *MI = MO.getParent();
SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)
: (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx);
if (InRange) {
unsigned SubRegIdx = MO.getSubReg();
- LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
+ LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
LastUseMask &= ~UseMask;
if (LastUseMask.none())
return LaneBitmask::getNone();
@@ -517,7 +519,9 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
+
+ const SIRegisterInfo *TRI =
+ MI->getMF()->getSubtarget<GCNSubtarget>().getRegisterInfo();
RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/true);
assert(RegOpers.DeadDefs.empty());
RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
@@ -558,8 +562,8 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
LastTrackedMI ? LIS.getInstructionIndex(*LastTrackedMI).getRegSlot()
: LIS.getMBBEndIdx(MI->getParent());
;
- LaneBitmask LastUseMask =
- findUseBetween(Reg, P.LaneMask, SlotIdx, CurrIdx, *MRI, &LIS, true);
+ LaneBitmask LastUseMask = findUseBetween(Reg, P.LaneMask, SlotIdx, CurrIdx,
+ *MRI, TRI, &LIS, true);
LastUseMask &= ~LiveAfter;
LaneBitmask LiveBefore = (LiveAfter | LastUseMask);
CurPressure.inc(Reg, LiveAfter, LiveBefore, *MRI);
@@ -733,7 +737,8 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
+ const SIRegisterInfo *TRI =
+ MI->getMF()->getSubtarget<GCNSubtarget>().getRegisterInfo();
RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/false);
RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
@@ -760,7 +765,7 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
}
LastUseMask =
- findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, &LIS);
+ findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);
if (LastUseMask.none())
continue;
>From 0415446afd03493a9347ef1482e9e117621a9fa4 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Wed, 21 Aug 2024 15:16:05 -0700
Subject: [PATCH 19/19] Review comments + revert to use_nodbg_operands
---
llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 16 +++++-----------
llvm/lib/Target/AMDGPU/GCNRegPressure.h | 4 ++--
2 files changed, 7 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 4374719645aec4..d66f67d0f49d9c 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -320,8 +320,6 @@ static LaneBitmask getLanesWithProperty(
}
const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
- // Be prepared for missing liveranges: We usually do not compute liveranges
- // for physical registers on targets with many registers (GPUs).
if (LR == nullptr)
return SafeDefault;
return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
@@ -336,11 +334,9 @@ static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
const SIRegisterInfo *TRI,
const LiveIntervals *LIS,
bool Upward = false) {
- for (const MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {
+ for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
if (MO.isUndef())
continue;
- if (!MO.readsReg())
- continue;
const MachineInstr *MI = MO.getParent();
SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)
@@ -512,7 +508,8 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
assert(CurPressure == getRegPressure(*MRI, LiveRegs));
}
-void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
+void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI,
+ const SIRegisterInfo *TRI) {
assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
SlotIndex SlotIdx = LIS.getInstructionIndex(*MI).getRegSlot();
@@ -520,8 +517,6 @@ void GCNUpwardRPTracker::bumpUpwardPressure(const MachineInstr *MI) {
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- const SIRegisterInfo *TRI =
- MI->getMF()->getSubtarget<GCNSubtarget>().getRegisterInfo();
RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/true);
assert(RegOpers.DeadDefs.empty());
RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
@@ -729,7 +724,8 @@ Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
});
}
-void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
+void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
+ const SIRegisterInfo *TRI) {
assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
SlotIndex SlotIdx;
@@ -737,8 +733,6 @@ void GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI) {
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
- const SIRegisterInfo *TRI =
- MI->getMF()->getSubtarget<GCNSubtarget>().getRegisterInfo();
RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/false);
RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 29ebeb1843c56c..3044672bb9c149 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -217,7 +217,7 @@ class GCNUpwardRPTracker : public GCNRPTracker {
/// to false allows for an externally managed iterator / program order.
void recede(const MachineInstr &MI);
- void bumpUpwardPressure(const MachineInstr *MI);
+ void bumpUpwardPressure(const MachineInstr *MI, const SIRegisterInfo *TRI);
/// \p returns whether the tracker's state after receding MI corresponds
/// to reported by LIS.
@@ -301,7 +301,7 @@ class GCNDownwardRPTracker : public GCNRPTracker {
MachineBasicBlock::const_iterator End,
const LiveRegSet *LiveRegsCopy = nullptr);
- void bumpDownwardPressure(const MachineInstr *MI);
+ void bumpDownwardPressure(const MachineInstr *MI, const SIRegisterInfo *TRI);
};
LaneBitmask getLiveLaneMask(unsigned Reg,
More information about the llvm-commits
mailing list