[llvm] r303548 - [AMDGPU] Fix incorrect register usage tracking in GCNUpwardTracker
Valery Pykhtin via llvm-commits
llvm-commits at lists.llvm.org
Mon May 22 06:09:41 PDT 2017
Author: vpykhtin
Date: Mon May 22 08:09:40 2017
New Revision: 303548
URL: http://llvm.org/viewvc/llvm-project?rev=303548&view=rev
Log:
[AMDGPU] Fix incorrect register usage tracking in GCNUpwardTracker
Differential revision: https://reviews.llvm.org/D33289
Modified:
llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp
llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h
Modified: llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp?rev=303548&r1=303547&r2=303548&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.cpp Mon May 22 08:09:40 2017
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "GCNRegPressure.h"
+#include "llvm/CodeGen/RegisterPressure.h"
using namespace llvm;
@@ -63,15 +64,6 @@ static bool isEqual(const GCNRPTracker::
return true;
}
-static GCNRPTracker::LiveRegSet
-stripEmpty(const GCNRPTracker::LiveRegSet &LR) {
- GCNRPTracker::LiveRegSet Res;
- for (const auto &P : LR) {
- if (P.second.any())
- Res.insert(P);
- }
- return Res;
-}
#endif
///////////////////////////////////////////////////////////////////////////////
@@ -185,6 +177,64 @@ void GCNRegPressure::print(raw_ostream &
}
#endif
+
+static LaneBitmask getDefRegMask(const MachineOperand &MO,
+ const MachineRegisterInfo &MRI) {
+ assert(MO.isDef() && MO.isReg() &&
+ TargetRegisterInfo::isVirtualRegister(MO.getReg()));
+
+ // We don't rely on read-undef flag because in case of tentative schedule
+ // tracking it isn't set correctly yet. This works correctly however since
+ // use mask has been tracked before using LIS.
+ return MO.getSubReg() == 0 ?
+ MRI.getMaxLaneMaskForVReg(MO.getReg()) :
+ MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
+}
+
+static LaneBitmask getUsedRegMask(const MachineOperand &MO,
+ const MachineRegisterInfo &MRI,
+ const LiveIntervals &LIS) {
+ assert(MO.isUse() && MO.isReg() &&
+ TargetRegisterInfo::isVirtualRegister(MO.getReg()));
+
+ if (auto SubReg = MO.getSubReg())
+ return MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(SubReg);
+
+ auto MaxMask = MRI.getMaxLaneMaskForVReg(MO.getReg());
+ if (MaxMask.getAsInteger() == 1) // cannot have subregs
+ return MaxMask;
+
+ // For a tentative schedule LIS isn't updated yet but livemask should remain
+ // the same on any schedule. Subreg defs can be reordered but they all must
+ // dominate uses anyway.
+ auto SI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
+ return getLiveLaneMask(MO.getReg(), SI, LIS, MRI);
+}
+
+SmallVector<RegisterMaskPair, 8> collectVirtualRegUses(const MachineInstr &MI,
+ const LiveIntervals &LIS,
+ const MachineRegisterInfo &MRI) {
+ SmallVector<RegisterMaskPair, 8> Res;
+ for (const auto &MO : MI.operands()) {
+ if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
+ continue;
+ if (!MO.isUse() || !MO.readsReg())
+ continue;
+
+ auto const UsedMask = getUsedRegMask(MO, MRI, LIS);
+
+ auto Reg = MO.getReg();
+ auto I = std::find_if(Res.begin(), Res.end(), [Reg](const RegisterMaskPair &RM) {
+ return RM.RegUnit == Reg;
+ });
+ if (I != Res.end())
+ I->LaneMask |= UsedMask;
+ else
+ Res.push_back(RegisterMaskPair(Reg, UsedMask));
+ }
+ return Res;
+}
+
///////////////////////////////////////////////////////////////////////////////
// GCNRPTracker
@@ -222,36 +272,6 @@ GCNRPTracker::LiveRegSet llvm::getLiveRe
return LiveRegs;
}
-LaneBitmask GCNRPTracker::getDefRegMask(const MachineOperand &MO) const {
- assert(MO.isDef() && MO.isReg() &&
- TargetRegisterInfo::isVirtualRegister(MO.getReg()));
-
- // We don't rely on read-undef flag because in case of tentative schedule
- // tracking it isn't set correctly yet. This works correctly however since
- // use mask has been tracked before using LIS.
- return MO.getSubReg() == 0 ?
- MRI->getMaxLaneMaskForVReg(MO.getReg()) :
- MRI->getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
-}
-
-LaneBitmask GCNRPTracker::getUsedRegMask(const MachineOperand &MO) const {
- assert(MO.isUse() && MO.isReg() &&
- TargetRegisterInfo::isVirtualRegister(MO.getReg()));
-
- if (auto SubReg = MO.getSubReg())
- return MRI->getTargetRegisterInfo()->getSubRegIndexLaneMask(SubReg);
-
- auto MaxMask = MRI->getMaxLaneMaskForVReg(MO.getReg());
- if (MaxMask.getAsInteger() == 1) // cannot have subregs
- return MaxMask;
-
- // For a tentative schedule LIS isn't updated yet but livemask should remain
- // the same on any schedule. Subreg defs can be reordered but they all must
- // dominate uses anyway.
- auto SI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
- return getLiveLaneMask(MO.getReg(), SI, LIS, *MRI);
-}
-
void GCNUpwardRPTracker::reset(const MachineInstr &MI,
const LiveRegSet *LiveRegsCopy) {
MRI = &MI.getParent()->getParent()->getRegInfo();
@@ -272,34 +292,40 @@ void GCNUpwardRPTracker::recede(const Ma
if (MI.isDebugValue())
return;
- // process all defs first to ensure early clobbers are handled correctly
- // iterating over operands() to catch implicit defs
- for (const auto &MO : MI.operands()) {
- if (!MO.isReg() || !MO.isDef() ||
- !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
+ auto const RegUses = collectVirtualRegUses(MI, LIS, *MRI);
+
+ // calc pressure at the MI (defs + uses)
+ auto AtMIPressure = CurPressure;
+ for (const auto &U : RegUses) {
+ auto LiveMask = LiveRegs[U.RegUnit];
+ AtMIPressure.inc(U.RegUnit, LiveMask, LiveMask | U.LaneMask, *MRI);
+ }
+ // update max pressure
+ MaxPressure = max(AtMIPressure, MaxPressure);
+
+ for (const auto &MO : MI.defs()) {
+ if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()) ||
+ MO.isDead())
continue;
auto Reg = MO.getReg();
- auto &LiveMask = LiveRegs[Reg];
+ auto I = LiveRegs.find(Reg);
+ if (I == LiveRegs.end())
+ continue;
+ auto &LiveMask = I->second;
auto PrevMask = LiveMask;
- LiveMask &= ~getDefRegMask(MO);
+ LiveMask &= ~getDefRegMask(MO, *MRI);
CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ if (LiveMask.none())
+ LiveRegs.erase(I);
}
-
- // then all uses
- for (const auto &MO : MI.uses()) {
- if (!MO.isReg() || !MO.readsReg() ||
- !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
- continue;
-
- auto Reg = MO.getReg();
- auto &LiveMask = LiveRegs[Reg];
+ for (const auto &U : RegUses) {
+ auto &LiveMask = LiveRegs[U.RegUnit];
auto PrevMask = LiveMask;
- LiveMask |= getUsedRegMask(MO);
- CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
+ LiveMask |= U.LaneMask;
+ CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI);
}
-
- MaxPressure = max(MaxPressure, CurPressure);
+ assert(CurPressure == getRegPressure(*MRI, LiveRegs));
}
bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
@@ -368,7 +394,7 @@ void GCNDownwardRPTracker::advanceToNext
continue;
auto &LiveMask = LiveRegs[Reg];
auto PrevMask = LiveMask;
- LiveMask |= getDefRegMask(MO);
+ LiveMask |= getDefRegMask(MO, *MRI);
CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
}
@@ -430,7 +456,7 @@ static void reportMismatch(const GCNRPTr
bool GCNUpwardRPTracker::isValid() const {
const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();
const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);
- const auto TrackedLR = stripEmpty(LiveRegs);
+ const auto &TrackedLR = LiveRegs;
if (!isEqual(LISLR, TrackedLR)) {
dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
Modified: llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h?rev=303548&r1=303547&r2=303548&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/GCNRegPressure.h Mon May 22 08:09:40 2017
@@ -98,8 +98,6 @@ protected:
const MachineInstr *LastTrackedMI = nullptr;
mutable const MachineRegisterInfo *MRI = nullptr;
GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
- LaneBitmask getDefRegMask(const MachineOperand &MO) const;
- LaneBitmask getUsedRegMask(const MachineOperand &MO) const;
public:
// live regs for the current state
const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
More information about the llvm-commits
mailing list