<div dir="ltr">This triggered -Wparentheses and I actually think the assert was wrong. Fixed in r196567, please take a look and double-check (as I am not familiar with this part of the codebase).</div><div class="gmail_extra">
<br><br><div class="gmail_quote">On Thu, Dec 5, 2013 at 6:56 PM, Andrew Trick <span dir="ltr"><<a href="mailto:atrick@apple.com" target="_blank">atrick@apple.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">
Author: atrick<br>
Date: Thu Dec 5 11:56:02 2013<br>
New Revision: 196517<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=196517&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=196517&view=rev</a><br>
Log:<br>
MI-Sched: Model "reserved" processor resources.<br>
<br>
This allows a target to use MI-Sched as an in-order scheduler that<br>
will model strict resource conflicts without defining a processor<br>
itinerary. Instead, the target can now use the new per-operand machine<br>
model and define in-order resources with BufferSize=0. For example,<br>
this would allow restricting the type of operations that can be formed<br>
into a dispatch group. (Normally NumMicroOps is sufficient to enforce<br>
dispatch groups).<br>
<br>
If the intent is to model latency in in-order pipeline, as opposed to<br>
resource conflicts, then a resource with BufferSize=1 should be<br>
defined instead.<br>
<br>
This feature is only casually tested as there are no in-tree targets<br>
using it yet. However, Hal will be experimenting with POWER7.<br>
<br>
Modified:<br>
llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h<br>
llvm/trunk/lib/CodeGen/MachineScheduler.cpp<br>
llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h?rev=196517&r1=196516&r2=196517&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h?rev=196517&r1=196516&r2=196517&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h Thu Dec 5 11:56:02 2013<br>
@@ -292,7 +292,8 @@ namespace llvm {<br>
bool isScheduleHigh : 1; // True if preferable to schedule high.<br>
bool isScheduleLow : 1; // True if preferable to schedule low.<br>
bool isCloned : 1; // True if this node has been cloned.<br>
- bool isUnbuffered : 1; // Reads an unbuffered resource.<br>
+ bool isUnbuffered : 1; // Uses an unbuffered resource.<br>
+ bool hasReservedResource : 1; // Uses a reserved resource.<br>
Sched::Preference SchedulingPref; // Scheduling preference.<br>
<br>
private:<br>
@@ -318,9 +319,9 @@ namespace llvm {<br>
hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),<br>
isAvailable(false), isScheduled(false), isScheduleHigh(false),<br>
isScheduleLow(false), isCloned(false), isUnbuffered(false),<br>
- SchedulingPref(Sched::None), isDepthCurrent(false),<br>
- isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),<br>
- BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
+ hasReservedResource(false), SchedulingPref(Sched::None),<br>
+ isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),<br>
+ TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
<br>
/// SUnit - Construct an SUnit for post-regalloc scheduling to represent<br>
/// a MachineInstr.<br>
@@ -333,9 +334,9 @@ namespace llvm {<br>
hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),<br>
isAvailable(false), isScheduled(false), isScheduleHigh(false),<br>
isScheduleLow(false), isCloned(false), isUnbuffered(false),<br>
- SchedulingPref(Sched::None), isDepthCurrent(false),<br>
- isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),<br>
- BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
+ hasReservedResource(false), SchedulingPref(Sched::None),<br>
+ isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),<br>
+ TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
<br>
/// SUnit - Construct a placeholder SUnit.<br>
SUnit()<br>
@@ -347,9 +348,9 @@ namespace llvm {<br>
hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),<br>
isAvailable(false), isScheduled(false), isScheduleHigh(false),<br>
isScheduleLow(false), isCloned(false), isUnbuffered(false),<br>
- SchedulingPref(Sched::None), isDepthCurrent(false),<br>
- isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),<br>
- BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
+ hasReservedResource(false), SchedulingPref(Sched::None),<br>
+ isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),<br>
+ TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}<br>
<br>
/// \brief Boundary nodes are placeholders for the boundary of the<br>
/// scheduling region.<br>
<br>
Modified: llvm/trunk/lib/CodeGen/MachineScheduler.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachineScheduler.cpp?rev=196517&r1=196516&r2=196517&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachineScheduler.cpp?rev=196517&r1=196516&r2=196517&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/MachineScheduler.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/MachineScheduler.cpp Thu Dec 5 11:56:02 2013<br>
@@ -1322,6 +1322,8 @@ void CopyConstrain::apply(ScheduleDAGMI<br>
// GenericScheduler - Implementation of the generic MachineSchedStrategy.<br>
//===----------------------------------------------------------------------===//<br>
<br>
+static const unsigned InvalidCycle = ~0U;<br>
+<br>
namespace {<br>
/// GenericScheduler shrinks the unscheduled zone using heuristics to balance<br>
/// the schedule.<br>
@@ -1491,6 +1493,10 @@ public:<br>
// Is the scheduled region resource limited vs. latency limited.<br>
bool IsResourceLimited;<br>
<br>
+ // Record the highest cycle at which each resource has been reserved by a<br>
+ // scheduled instruction.<br>
+ SmallVector<unsigned, 16> ReservedCycles;<br>
+<br>
#ifndef NDEBUG<br>
// Remember the greatest operand latency as an upper bound on the number of<br>
// times we should retry the pending queue because of a hazard.<br>
@@ -1518,6 +1524,7 @@ public:<br>
MaxExecutedResCount = 0;<br>
ZoneCritResIdx = 0;<br>
IsResourceLimited = false;<br>
+ ReservedCycles.clear();<br>
#ifndef NDEBUG<br>
MaxObservedLatency = 0;<br>
#endif<br>
@@ -1587,6 +1594,8 @@ public:<br>
/// cycle.<br>
unsigned getLatencyStallCycles(SUnit *SU);<br>
<br>
+ unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles);<br>
+<br>
bool checkHazard(SUnit *SU);<br>
<br>
unsigned findMaxLatency(ArrayRef<SUnit*> ReadySUs);<br>
@@ -1708,8 +1717,10 @@ init(ScheduleDAGMI *dag, const TargetSch<br>
DAG = dag;<br>
SchedModel = smodel;<br>
Rem = rem;<br>
- if (SchedModel->hasInstrSchedModel())<br>
+ if (SchedModel->hasInstrSchedModel()) {<br>
ExecutedResCounts.resize(SchedModel->getNumProcResourceKinds());<br>
+ ReservedCycles.resize(SchedModel->getNumProcResourceKinds(), InvalidCycle);<br>
+ }<br>
}<br>
<br>
/// Initialize the per-region scheduling policy.<br>
@@ -1890,6 +1901,20 @@ unsigned GenericScheduler::SchedBoundary<br>
return 0;<br>
}<br>
<br>
+/// Compute the next cycle at which the given processor resource can be<br>
+/// scheduled.<br>
+unsigned GenericScheduler::SchedBoundary::<br>
+getNextResourceCycle(unsigned PIdx, unsigned Cycles) {<br>
+ unsigned NextUnreserved = ReservedCycles[PIdx];<br>
+ // If this resource has never been used, always return cycle zero.<br>
+ if (NextUnreserved == InvalidCycle)<br>
+ return 0;<br>
+ // For bottom-up scheduling add the cycles needed for the current operation.<br>
+ if (!isTop())<br>
+ NextUnreserved += Cycles;<br>
+ return NextUnreserved;<br>
+}<br>
+<br>
/// Does this SU have a hazard within the current instruction group.<br>
///<br>
/// The scheduler supports two modes of hazard recognition. The first is the<br>
@@ -1913,6 +1938,15 @@ bool GenericScheduler::SchedBoundary::ch<br>
<< SchedModel->getNumMicroOps(SU->getInstr()) << '\n');<br>
return true;<br>
}<br>
+ if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {<br>
+ const MCSchedClassDesc *SC = DAG->getSchedClass(SU);<br>
+ for (TargetSchedModel::ProcResIter<br>
+ PI = SchedModel->getWriteProcResBegin(SC),<br>
+ PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {<br>
+ if (getNextResourceCycle(PI->ProcResourceIdx, PI->Cycles) > CurrCycle)<br>
+ return true;<br>
+ }<br>
+ }<br>
return false;<br>
}<br>
<br>
@@ -2097,7 +2131,7 @@ void GenericScheduler::SchedBoundary::in<br>
/// \return the next cycle at which the instruction may execute without<br>
/// oversubscribing resources.<br>
unsigned GenericScheduler::SchedBoundary::<br>
-countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle) {<br>
+countResource(unsigned PIdx, unsigned Cycles, unsigned NextCycle) {<br>
unsigned Factor = SchedModel->getResourceFactor(PIdx);<br>
unsigned Count = Factor * Cycles;<br>
DEBUG(dbgs() << " " << getResourceName(PIdx)<br>
@@ -2116,8 +2150,14 @@ countResource(unsigned PIdx, unsigned Cy<br>
<< getResourceName(PIdx) << ": "<br>
<< getResourceCount(PIdx) / SchedModel->getLatencyFactor() << "c\n");<br>
}<br>
- // TODO: We don't yet model reserved resources. It's not hard though.<br>
- return CurrCycle;<br>
+ // For reserved resources, record the highest cycle using the resource.<br>
+ unsigned NextAvailable = getNextResourceCycle(PIdx, Cycles);<br>
+ if (NextAvailable > CurrCycle) {<br>
+ DEBUG(dbgs() << " Resource conflict: "<br>
+ << SchedModel->getProcResource(PIdx)->Name << " reserved until @"<br>
+ << NextAvailable << "\n");<br>
+ }<br>
+ return NextAvailable;<br>
}<br>
<br>
/// Move the boundary of scheduled code by one SUnit.<br>
@@ -2131,25 +2171,17 @@ void GenericScheduler::SchedBoundary::bu<br>
}<br>
HazardRec->EmitInstruction(SU);<br>
}<br>
+ // checkHazard should prevent scheduling multiple instructions per cycle that<br>
+ // exceed the issue width.<br>
const MCSchedClassDesc *SC = DAG->getSchedClass(SU);<br>
unsigned IncMOps = SchedModel->getNumMicroOps(SU->getInstr());<br>
- CurrMOps += IncMOps;<br>
- // checkHazard prevents scheduling multiple instructions per cycle that exceed<br>
- // issue width. However, we commonly reach the maximum. In this case<br>
- // opportunistically bump the cycle to avoid uselessly checking everything in<br>
- // the readyQ. Furthermore, a single instruction may produce more than one<br>
- // cycle's worth of micro-ops.<br>
- //<br>
- // TODO: Also check if this SU must end a dispatch group.<br>
- unsigned NextCycle = CurrCycle;<br>
- if (CurrMOps >= SchedModel->getIssueWidth()) {<br>
- ++NextCycle;<br>
- DEBUG(dbgs() << " *** Max MOps " << CurrMOps<br>
- << " at cycle " << CurrCycle << '\n');<br>
- }<br>
+ assert(CurrMOps == 0 || (CurrMOps + IncMOps) <= SchedModel->getIssueWidth() &&<br>
+ "Cannot scheduling this instructions MicroOps in the current cycle.");<br>
+<br>
unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle);<br>
DEBUG(dbgs() << " Ready @" << ReadyCycle << "c\n");<br>
<br>
+ unsigned NextCycle = CurrCycle;<br>
switch (SchedModel->getMicroOpBufferSize()) {<br>
case 0:<br>
assert(ReadyCycle <= CurrCycle && "Broken PendingQueue");<br>
@@ -2194,10 +2226,23 @@ void GenericScheduler::SchedBoundary::bu<br>
PI = SchedModel->getWriteProcResBegin(SC),<br>
PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {<br>
unsigned RCycle =<br>
- countResource(PI->ProcResourceIdx, PI->Cycles, ReadyCycle);<br>
+ countResource(PI->ProcResourceIdx, PI->Cycles, NextCycle);<br>
if (RCycle > NextCycle)<br>
NextCycle = RCycle;<br>
}<br>
+ if (SU->hasReservedResource) {<br>
+ // For reserved resources, record the highest cycle using the resource.<br>
+ // For top-down scheduling, this is the cycle in which we schedule this<br>
+ // instruction plus the number of cycles the operations reserves the<br>
+ // resource. For bottom-up is it simply the instruction's cycle.<br>
+ for (TargetSchedModel::ProcResIter<br>
+ PI = SchedModel->getWriteProcResBegin(SC),<br>
+ PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {<br>
+ unsigned PIdx = PI->ProcResourceIdx;<br>
+ if (SchedModel->getProcResource(PIdx)->BufferSize == 0)<br>
+ ReservedCycles[PIdx] = isTop() ? NextCycle + PI->Cycles : NextCycle;<br>
+ }<br>
+ }<br>
}<br>
// Update ExpectedLatency and DependentLatency.<br>
unsigned &TopLatency = isTop() ? ExpectedLatency : DependentLatency;<br>
@@ -2224,6 +2269,16 @@ void GenericScheduler::SchedBoundary::bu<br>
(int)(getCriticalCount() - (getScheduledLatency() * LFactor))<br>
> (int)LFactor;<br>
}<br>
+ // Update CurrMOps after calling bumpCycle to handle stalls, since bumpCycle<br>
+ // resets CurrMOps. Loop to handle instructions with more MOps than issue in<br>
+ // one cycle. Since we commonly reach the max MOps here, opportunistically<br>
+ // bump the cycle to avoid uselessly checking everything in the readyQ.<br>
+ CurrMOps += IncMOps;<br>
+ while (CurrMOps >= SchedModel->getIssueWidth()) {<br>
+ bumpCycle(++NextCycle);<br>
+ DEBUG(dbgs() << " *** Max MOps " << CurrMOps<br>
+ << " at cycle " << CurrCycle << '\n');<br>
+ }<br>
DEBUG(dumpScheduledState());<br>
}<br>
<br>
<br>
Modified: llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp?rev=196517&r1=196516&r2=196517&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp?rev=196517&r1=196516&r2=196517&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp Thu Dec 5 11:56:02 2013<br>
@@ -697,9 +697,15 @@ void ScheduleDAGInstrs::initSUnits() {<br>
for (TargetSchedModel::ProcResIter<br>
PI = SchedModel.getWriteProcResBegin(SC),<br>
PE = SchedModel.getWriteProcResEnd(SC); PI != PE; ++PI) {<br>
- if (SchedModel.getProcResource(PI->ProcResourceIdx)->BufferSize == 1) {<br>
+ switch (SchedModel.getProcResource(PI->ProcResourceIdx)->BufferSize) {<br>
+ case 0:<br>
+ SU->hasReservedResource = true;<br>
+ break;<br>
+ case 1:<br>
SU->isUnbuffered = true;<br>
break;<br>
+ default:<br>
+ break;<br>
}<br>
}<br>
}<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div>