[llvm] r189643 - mi-sched: update PressureDiffs on-the-fly for liveness.
Andrew Trick
atrick at apple.com
Thu Aug 29 21:36:57 PDT 2013
Author: atrick
Date: Thu Aug 29 23:36:57 2013
New Revision: 189643
URL: http://llvm.org/viewvc/llvm-project?rev=189643&view=rev
Log:
mi-sched: update PressureDiffs on-the-fly for liveness.
This removes all expensive pressure tracking logic from the scheduling
critical path of node comparison.
Modified:
llvm/trunk/include/llvm/CodeGen/LiveInterval.h
llvm/trunk/include/llvm/CodeGen/MachineScheduler.h
llvm/trunk/include/llvm/CodeGen/RegisterPressure.h
llvm/trunk/lib/CodeGen/MachineScheduler.cpp
llvm/trunk/lib/CodeGen/RegisterPressure.cpp
llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp
Modified: llvm/trunk/include/llvm/CodeGen/LiveInterval.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LiveInterval.h?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/LiveInterval.h (original)
+++ llvm/trunk/include/llvm/CodeGen/LiveInterval.h Thu Aug 29 23:36:57 2013
@@ -296,6 +296,12 @@ namespace llvm {
return r != end() && r->end.getBaseIndex() == BaseIdx;
}
+ /// Return true if a live range starts at the instruction at this index.
+ bool isDefinedByInstr(SlotIndex index) const {
+ const_iterator r = find(index.getDeadSlot());
+ return r != end() && r->end.getBaseIndex() == index.getBaseIndex();
+ }
+
/// getLiveRangeContaining - Return the live range that contains the
/// specified index, or null if there is none.
const LiveRange *getLiveRangeContaining(SlotIndex Idx) const {
Modified: llvm/trunk/include/llvm/CodeGen/MachineScheduler.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/MachineScheduler.h?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/MachineScheduler.h (original)
+++ llvm/trunk/include/llvm/CodeGen/MachineScheduler.h Thu Aug 29 23:36:57 2013
@@ -221,7 +221,9 @@ protected:
MachineBasicBlock::iterator LiveRegionEnd;
- // Map each SU to its summary of pressure changes.
+ // Map each SU to its summary of pressure changes. This array is updated for
+ // liveness during bottom-up scheduling. Top-down scheduling may proceed but
+ // has no affect on the pressure diffs.
PressureDiffs SUPressureDiffs;
/// Register pressure in this region computed by initRegPressure.
@@ -376,6 +378,8 @@ protected:
void initRegPressure();
+ void updatePressureDiffs(ArrayRef<unsigned> LiveUses);
+
void updateScheduledPressure(const std::vector<unsigned> &NewMaxPressure);
bool checkSchedLimit();
Modified: llvm/trunk/include/llvm/CodeGen/RegisterPressure.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/RegisterPressure.h?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/RegisterPressure.h (original)
+++ llvm/trunk/include/llvm/CodeGen/RegisterPressure.h Thu Aug 29 23:36:57 2013
@@ -311,7 +311,7 @@ public:
SlotIndex getCurrSlot() const;
/// Recede across the previous instruction.
- bool recede(PressureDiff *PDiff = 0);
+ bool recede(SmallVectorImpl<unsigned> *LiveUses = 0, PressureDiff *PDiff = 0);
/// Advance across the current instruction.
bool advance();
Modified: llvm/trunk/lib/CodeGen/MachineScheduler.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachineScheduler.cpp?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/MachineScheduler.cpp (original)
+++ llvm/trunk/lib/CodeGen/MachineScheduler.cpp Thu Aug 29 23:36:57 2013
@@ -156,8 +156,9 @@ static ScheduleDAGInstrs *createConvergi
/// Decrement this iterator until reaching the top or a non-debug instr.
-static MachineBasicBlock::iterator
-priorNonDebug(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Beg) {
+static MachineBasicBlock::const_iterator
+priorNonDebug(MachineBasicBlock::const_iterator I,
+ MachineBasicBlock::const_iterator Beg) {
assert(I != Beg && "reached the top of the region, cannot decrement");
while (--I != Beg) {
if (!I->isDebugValue())
@@ -166,6 +167,14 @@ priorNonDebug(MachineBasicBlock::iterato
return I;
}
+/// Non-const version.
+static MachineBasicBlock::iterator
+priorNonDebug(MachineBasicBlock::iterator I,
+ MachineBasicBlock::const_iterator Beg) {
+ return const_cast<MachineInstr*>(
+ &*priorNonDebug(MachineBasicBlock::const_iterator(I), Beg));
+}
+
/// If this iterator is a debug value, increment until reaching the End or a
/// non-debug instruction.
static MachineBasicBlock::iterator
@@ -488,9 +497,16 @@ void ScheduleDAGMI::initRegPressure() {
dumpRegSetPressure(BotRPTracker.getLiveThru(), TRI));
};
+ // For each live out vreg reduce the pressure change associated with other
+ // uses of the same vreg below the live-out reaching def.
+ updatePressureDiffs(RPTracker.getPressure().LiveOutRegs);
+
// Account for liveness generated by the region boundary.
- if (LiveRegionEnd != RegionEnd)
- BotRPTracker.recede();
+ if (LiveRegionEnd != RegionEnd) {
+ SmallVector<unsigned, 8> LiveUses;
+ BotRPTracker.recede(&LiveUses);
+ updatePressureDiffs(LiveUses);
+ }
assert(BotRPTracker.getPos() == RegionEnd && "Can't find the region bottom");
@@ -535,6 +551,42 @@ updateScheduledPressure(const std::vecto
});
}
+/// Update the PressureDiff array for liveness after scheduling this
+/// instruction.
+void ScheduleDAGMI::updatePressureDiffs(ArrayRef<unsigned> LiveUses) {
+ for (unsigned LUIdx = 0, LUEnd = LiveUses.size(); LUIdx != LUEnd; ++LUIdx) {
+ /// FIXME: Currently assuming single-use physregs.
+ unsigned Reg = LiveUses[LUIdx];
+ if (!TRI->isVirtualRegister(Reg))
+ continue;
+ // This may be called before CurrentBottom has been initialized. However,
+ // BotRPTracker must have a valid position. We want the value live into the
+ // instruction or live out of the block, so ask for the previous
+ // instruction's live-out.
+ const LiveInterval &LI = LIS->getInterval(Reg);
+ VNInfo *VNI;
+ if (BotRPTracker.getPos() == BB->end())
+ VNI = LI.getVNInfoBefore(LIS->getMBBEndIdx(BB));
+ else {
+ LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(BotRPTracker.getPos()));
+ VNI = LRQ.valueIn();
+ }
+ // RegisterPressureTracker guarantees that readsReg is true for LiveUses.
+ assert(VNI && "No live value at use.");
+ for (VReg2UseMap::iterator
+ UI = VRegUses.find(Reg); UI != VRegUses.end(); ++UI) {
+ SUnit *SU = UI->SU;
+ // If this use comes before the reaching def, it cannot be a last use, so
+ // descrease its pressure change.
+ if (!SU->isScheduled && SU != &ExitSU) {
+ LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(SU->getInstr()));
+ if (LRQ.valueIn() == VNI)
+ getPressureDiff(SU).addPressureChange(Reg, true, &MRI);
+ }
+ }
+ }
+}
+
/// schedule - Called back from MachineScheduler::runOnMachineFunction
/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
/// only includes instructions that have DAG nodes, not scheduling boundaries.
@@ -794,8 +846,10 @@ void ScheduleDAGMI::scheduleMI(SUnit *SU
CurrentBottom = MI;
}
// Update bottom scheduled pressure.
- BotRPTracker.recede();
+ SmallVector<unsigned, 8> LiveUses;
+ BotRPTracker.recede(&LiveUses);
assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
+ updatePressureDiffs(LiveUses);
updateScheduledPressure(BotRPTracker.getPressure().MaxSetPressure);
}
}
Modified: llvm/trunk/lib/CodeGen/RegisterPressure.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegisterPressure.cpp?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/RegisterPressure.cpp (original)
+++ llvm/trunk/lib/CodeGen/RegisterPressure.cpp Thu Aug 29 23:36:57 2013
@@ -399,10 +399,9 @@ static void collectPDiff(PressureDiff &P
const MachineRegisterInfo *MRI) {
assert(!PDiff.begin()->isValid() && "stale PDiff");
- for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i) {
- if (!containsReg(RegOpers.Uses, RegOpers.Defs[i]))
- PDiff.addPressureChange(RegOpers.Defs[i], true, MRI);
- }
+ for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i)
+ PDiff.addPressureChange(RegOpers.Defs[i], true, MRI);
+
for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i)
PDiff.addPressureChange(RegOpers.Uses[i], false, MRI);
}
@@ -437,9 +436,13 @@ void RegPressureTracker::discoverLiveOut
increaseSetPressure(P.MaxSetPressure, MRI->getPressureSets(Reg));
}
-/// Recede across the previous instruction.
-/// Record the pressure difference if it is provided.
-bool RegPressureTracker::recede(PressureDiff *PDiff) {
+/// Recede across the previous instruction. If LiveUses is provided, record any
+/// RegUnits that are made live by the current instruction's uses. This includes
+/// registers that are both defined and used by the instruction. If a pressure
+/// difference pointer is provided record the changes is pressure caused by this
+/// instruction independent of liveness.
+bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
+ PressureDiff *PDiff) {
// Check for the top of the analyzable region.
if (CurrPos == MBB->begin()) {
closeRegion();
@@ -496,11 +499,16 @@ bool RegPressureTracker::recede(Pressure
// Adjust liveouts if LiveIntervals are available.
if (RequireIntervals) {
const LiveInterval *LI = getInterval(Reg);
- if (LI && !LI->isKilledAtInstr(SlotIdx))
- discoverLiveOut(Reg);
+ // Check if this LR is killed and not redefined here.
+ if (LI && !LI->isKilledAtInstr(SlotIdx)
+ && !LI->isDefinedByInstr(SlotIdx)) {
+ discoverLiveOut(Reg);
+ }
}
increaseRegPressure(Reg);
LiveRegs.insert(Reg);
+ if (LiveUses && !containsReg(*LiveUses, Reg))
+ LiveUses->push_back(Reg);
}
}
if (TrackUntiedDefs) {
@@ -773,22 +781,10 @@ getMaxUpwardPressureDelta(const MachineI
/// @param MaxPressureLimit Is the max pressure within the region, not
/// necessarily at the current position.
void RegPressureTracker::
-getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff1,
+getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
RegPressureDelta &Delta,
ArrayRef<PressureChange> CriticalPSets,
ArrayRef<unsigned> MaxPressureLimit) const {
- RegisterOperands RegOpers(TRI, MRI, /*IgnoreDead=*/true);
- collectOperands(MI, RegOpers);
-
- // Decrease the pressure change for live uses.
- PressureDiff PDiff = PDiff1;
- for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i) {
- if (LiveRegs.contains(RegOpers.Uses[i]))
- PDiff.addPressureChange(RegOpers.Uses[i], true, MRI);
- }
-
- // Now directly query pressure from PDiff. Everything above this can be
- // cached and updated independent of the query.
unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
for (PressureDiff::const_iterator
PDiffI = PDiff.begin(), PDiffE = PDiff.end();
Modified: llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp?rev=189643&r1=189642&r2=189643&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp (original)
+++ llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp Thu Aug 29 23:36:57 2013
@@ -408,7 +408,13 @@ void ScheduleDAGInstrs::addVRegUseDeps(S
unsigned Reg = MI->getOperand(OperIdx).getReg();
// Record this local VReg use.
- VRegUses.insert(VReg2SUnit(Reg, SU));
+ VReg2UseMap::iterator UI = VRegUses.find(Reg);
+ for (; UI != VRegUses.end(); ++UI) {
+ if (UI->SU == SU)
+ break;
+ }
+ if (UI == VRegUses.end())
+ VRegUses.insert(VReg2SUnit(Reg, SU));
// Lookup this operand's reaching definition.
assert(LIS && "vreg dependencies requires LiveIntervals");
@@ -755,7 +761,7 @@ void ScheduleDAGInstrs::buildSchedGraph(
if (RPTracker) {
PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : 0;
- RPTracker->recede(PDiff);
+ RPTracker->recede(/*LiveUses=*/0, PDiff);
assert(RPTracker->getPos() == prior(MII) && "RPTracker can't find MI");
}
More information about the llvm-commits
mailing list