[llvm-commits] [llvm] r122541 - in /llvm/trunk: include/llvm/CodeGen/ include/llvm/Target/ lib/CodeGen/ lib/CodeGen/SelectionDAG/ lib/Target/ARM/ lib/Target/CellSPU/ lib/Target/PowerPC/
Andrew Trick
atrick at apple.com
Thu Dec 23 21:03:26 PST 2010
Author: atrick
Date: Thu Dec 23 23:03:26 2010
New Revision: 122541
URL: http://llvm.org/viewvc/llvm-project?rev=122541&view=rev
Log:
Various bits of framework needed for precise machine-level selection
DAG scheduling during isel. Most new functionality is currently
guarded by -enable-sched-cycles and -enable-sched-hazard.
Added InstrItineraryData::IssueWidth field, currently derived from
ARM itineraries, but could be initialized differently on other targets.
Added ScheduleHazardRecognizer::MaxLookAhead to indicate whether it is
active, and if so how many cycles of state it holds.
Added SchedulingPriorityQueue::HasReadyFilter to allowing gating entry
into the scheduler's available queue.
ScoreboardHazardRecognizer now accesses the ScheduleDAG in order to
get information about it's SUnits, provides RecedeCycle for bottom-up
scheduling, correctly computes scoreboard depth, tracks IssueCount, and
considers potential stall cycles when checking for hazards.
ScheduleDAGRRList now models machine cycles and hazards (under
flags). It tracks MinAvailableCycle, drives the hazard recognizer and
priority queue's ready filter, manages a new PendingQueue, properly
accounts for stall cycles, etc.
Modified:
llvm/trunk/include/llvm/CodeGen/LatencyPriorityQueue.h
llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h
llvm/trunk/include/llvm/CodeGen/ScheduleHazardRecognizer.h
llvm/trunk/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
llvm/trunk/include/llvm/CodeGen/SelectionDAGISel.h
llvm/trunk/include/llvm/Target/TargetInstrInfo.h
llvm/trunk/include/llvm/Target/TargetInstrItineraries.h
llvm/trunk/lib/CodeGen/LatencyPriorityQueue.cpp
llvm/trunk/lib/CodeGen/PostRASchedulerList.cpp
llvm/trunk/lib/CodeGen/ScheduleDAG.cpp
llvm/trunk/lib/CodeGen/ScoreboardHazardRecognizer.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
llvm/trunk/lib/CodeGen/TargetInstrInfoImpl.cpp
llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.cpp
llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.h
llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.cpp
llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.h
llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp
llvm/trunk/lib/Target/ARM/ARMSubtarget.h
llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.cpp
llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.h
llvm/trunk/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.cpp
llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.h
llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.cpp
llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.h
llvm/trunk/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.cpp
llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.h
Modified: llvm/trunk/include/llvm/CodeGen/LatencyPriorityQueue.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LatencyPriorityQueue.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/LatencyPriorityQueue.h (original)
+++ llvm/trunk/include/llvm/CodeGen/LatencyPriorityQueue.h Thu Dec 23 23:03:26 2010
@@ -47,6 +47,8 @@
LatencyPriorityQueue() : Picker(this) {
}
+ bool isBottomUp() const { return false; }
+
void initNodes(std::vector<SUnit> &sunits) {
SUnits = &sunits;
NumNodesSolelyBlocking.resize(SUnits->size(), 0);
@@ -81,6 +83,8 @@
virtual void remove(SUnit *SU);
+ virtual void dump(ScheduleDAG* DAG) const;
+
// ScheduledNode - As nodes are scheduled, we look to see if there are any
// successor nodes that have a single unscheduled predecessor. If so, that
// single predecessor has a higher priority, since scheduling it will make
Modified: llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h (original)
+++ llvm/trunk/include/llvm/CodeGen/ScheduleDAG.h Thu Dec 23 23:03:26 2010
@@ -326,6 +326,10 @@
return Node;
}
+ /// isInstr - Return true if this SUnit refers to a machine instruction as
+ /// opposed to an SDNode.
+ bool isInstr() const { return !Node; }
+
/// setInstr - Assign the instruction for the SUnit.
/// This may be used during post-regalloc scheduling.
void setInstr(MachineInstr *MI) {
@@ -421,16 +425,27 @@
///
class SchedulingPriorityQueue {
unsigned CurCycle;
+ bool HasReadyFilter;
public:
- SchedulingPriorityQueue() : CurCycle(0) {}
+ SchedulingPriorityQueue(bool rf = false):
+ CurCycle(0), HasReadyFilter(rf) {}
virtual ~SchedulingPriorityQueue() {}
+ virtual bool isBottomUp() const = 0;
+
virtual void initNodes(std::vector<SUnit> &SUnits) = 0;
virtual void addNode(const SUnit *SU) = 0;
virtual void updateNode(const SUnit *SU) = 0;
virtual void releaseState() = 0;
virtual bool empty() const = 0;
+
+ bool hasReadyFilter() const { return HasReadyFilter; }
+
+ virtual bool isReady(SUnit *U) const {
+ assert(!HasReadyFilter && "The ready filter must override isReady()");
+ return true;
+ }
virtual void push(SUnit *U) = 0;
void push_all(const std::vector<SUnit *> &Nodes) {
@@ -443,6 +458,8 @@
virtual void remove(SUnit *SU) = 0;
+ virtual void dump(ScheduleDAG *DAG) const {}
+
/// ScheduledNode - As each node is scheduled, this method is invoked. This
/// allows the priority function to adjust the priority of related
/// unscheduled nodes, for example.
@@ -479,6 +496,13 @@
virtual ~ScheduleDAG();
+ /// getInstrDesc - Return the TargetInstrDesc of this SUnit.
+ /// Return NULL for SDNodes without a machine opcode.
+ const TargetInstrDesc *getInstrDesc(const SUnit *SU) const {
+ if (SU->isInstr()) return &SU->getInstr()->getDesc();
+ return getNodeDesc(SU->getNode());
+ }
+
/// viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered
/// using 'dot'.
///
@@ -542,6 +566,10 @@
void EmitNoop();
void EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, unsigned> &VRBaseMap);
+
+ private:
+ // Return the TargetInstrDesc of this SDNode or NULL.
+ const TargetInstrDesc *getNodeDesc(const SDNode *Node) const;
};
class SUnitIterator : public std::iterator<std::forward_iterator_tag,
Modified: llvm/trunk/include/llvm/CodeGen/ScheduleHazardRecognizer.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ScheduleHazardRecognizer.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/ScheduleHazardRecognizer.h (original)
+++ llvm/trunk/include/llvm/CodeGen/ScheduleHazardRecognizer.h Thu Dec 23 23:03:26 2010
@@ -23,7 +23,15 @@
/// issued this cycle, and whether or not a noop needs to be inserted to handle
/// the hazard.
class ScheduleHazardRecognizer {
+protected:
+ /// MaxLookAhead - Indicate the number of cycles in the scoreboard
+ /// state. Important to restore the state after backtracking. Additionally,
+ /// MaxLookAhead=0 identifies a fake recognizer, allowing the client to
+ /// bypass virtual calls. Currently the PostRA scheduler ignores it.
+ unsigned MaxLookAhead;
+
public:
+ ScheduleHazardRecognizer(): MaxLookAhead(0) {}
virtual ~ScheduleHazardRecognizer();
enum HazardType {
@@ -32,6 +40,12 @@
NoopHazard // This instruction can't be emitted, and needs noops.
};
+ unsigned getMaxLookAhead() const { return MaxLookAhead; }
+
+ /// atIssueLimit - Return true if no more instructions may be issued in this
+ /// cycle.
+ virtual bool atIssueLimit() const { return false; }
+
/// getHazardType - Return the hazard type of emitting this node. There are
/// three possible results. Either:
/// * NoHazard: it is legal to issue this instruction on this cycle.
@@ -39,7 +53,7 @@
/// other instruction is available, issue it first.
/// * NoopHazard: issuing this instruction would break the program. If
/// some other instruction can be issued, do so, otherwise issue a noop.
- virtual HazardType getHazardType(SUnit *) {
+ virtual HazardType getHazardType(SUnit *m, int Stalls) {
return NoHazard;
}
@@ -52,12 +66,18 @@
/// emitted, to advance the hazard state.
virtual void EmitInstruction(SUnit *) {}
- /// AdvanceCycle - This callback is invoked when no instructions can be
- /// issued on this cycle without a hazard. This should increment the
+ /// AdvanceCycle - This callback is invoked whenever the next top-down
+ /// instruction to be scheduled cannot issue in the current cycle, either
+ /// because of latency or resource conflicts. This should increment the
/// internal state of the hazard recognizer so that previously "Hazard"
/// instructions will now not be hazards.
virtual void AdvanceCycle() {}
+ /// RecedeCycle - This callback is invoked whenever the next bottom-up
+ /// instruction to be scheduled cannot issue in the current cycle, either
+ /// because of latency or resource conflicts.
+ virtual void RecedeCycle() {}
+
/// EmitNoop - This callback is invoked when a noop was added to the
/// instruction stream.
virtual void EmitNoop() {
Modified: llvm/trunk/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ScoreboardHazardRecognizer.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/ScoreboardHazardRecognizer.h (original)
+++ llvm/trunk/include/llvm/CodeGen/ScoreboardHazardRecognizer.h Thu Dec 23 23:03:26 2010
@@ -26,6 +26,8 @@
namespace llvm {
class InstrItineraryData;
+class TargetInstrDesc;
+class ScheduleDAG;
class SUnit;
class ScoreboardHazardRecognizer : public ScheduleHazardRecognizer {
@@ -84,16 +86,38 @@
void dump() const;
};
+#ifndef NDEBUG
+ // Support for tracing ScoreboardHazardRecognizer as a component within
+ // another module. Follows the current thread-unsafe model of tracing.
+ static const char *DebugType;
+#endif
+
// Itinerary data for the target.
const InstrItineraryData *ItinData;
+ const ScheduleDAG *DAG;
+
+ /// IssueWidth - Max issue per cycle. 0=Unknown.
+ unsigned IssueWidth;
+
+ /// IssueCount - Count instructions issued in this cycle.
+ unsigned IssueCount;
+
Scoreboard ReservedScoreboard;
Scoreboard RequiredScoreboard;
public:
- ScoreboardHazardRecognizer(const InstrItineraryData *ItinData);
-
- virtual HazardType getHazardType(SUnit *SU);
+ ScoreboardHazardRecognizer(const InstrItineraryData *ItinData,
+ const ScheduleDAG *DAG,
+ const char *ParentDebugType = "");
+
+ /// atIssueLimit - Return true if no more instructions may be issued in this
+ /// cycle.
+ virtual bool atIssueLimit() const;
+
+ // Stalls provides an cycle offset at which SU will be scheduled. It will be
+ // negative for bottom-up scheduling.
+ virtual HazardType getHazardType(SUnit *SU, int Stalls);
virtual void Reset();
virtual void EmitInstruction(SUnit *SU);
virtual void AdvanceCycle();
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAGISel.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGISel.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/SelectionDAGISel.h (original)
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAGISel.h Thu Dec 23 23:03:26 2010
@@ -98,11 +98,6 @@
CodeGenOpt::Level OptLevel,
bool IgnoreChains = false);
- /// CreateTargetHazardRecognizer - Return a newly allocated hazard recognizer
- /// to use for this target when scheduling the DAG.
- virtual ScheduleHazardRecognizer *CreateTargetHazardRecognizer();
-
-
// Opcodes used by the DAG state machine:
enum BuiltinOpcodes {
OPC_Scope,
Modified: llvm/trunk/include/llvm/Target/TargetInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetInstrInfo.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetInstrInfo.h (original)
+++ llvm/trunk/include/llvm/Target/TargetInstrInfo.h Thu Dec 23 23:03:26 2010
@@ -29,6 +29,7 @@
class SDNode;
class ScheduleHazardRecognizer;
class SelectionDAG;
+class ScheduleDAG;
class TargetRegisterClass;
class TargetRegisterInfo;
@@ -563,11 +564,19 @@
virtual unsigned getInlineAsmLength(const char *Str,
const MCAsmInfo &MAI) const;
- /// CreateTargetHazardRecognizer - Allocate and return a hazard recognizer
- /// to use for this target when scheduling the machine instructions after
- /// register allocation.
+ /// CreateTargetPreRAHazardRecognizer - Allocate and return a hazard
+ /// recognizer to use for this target when scheduling the machine instructions
+ /// before register allocation.
virtual ScheduleHazardRecognizer*
- CreateTargetPostRAHazardRecognizer(const InstrItineraryData*) const = 0;
+ CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const = 0;
+
+ /// CreateTargetPostRAHazardRecognizer - Allocate and return a hazard
+ /// recognizer to use for this target when scheduling the machine instructions
+ /// after register allocation.
+ virtual ScheduleHazardRecognizer*
+ CreateTargetPostRAHazardRecognizer(const InstrItineraryData*,
+ const ScheduleDAG *DAG) const = 0;
/// AnalyzeCompare - For a comparison instruction, return the source register
/// in SrcReg and the value it compares against in CmpValue. Return true if
@@ -674,7 +683,11 @@
const MachineFunction &MF) const;
virtual ScheduleHazardRecognizer *
- CreateTargetPostRAHazardRecognizer(const InstrItineraryData*) const;
+ CreateTargetHazardRecognizer(const TargetMachine*, const ScheduleDAG*) const;
+
+ virtual ScheduleHazardRecognizer *
+ CreateTargetPostRAHazardRecognizer(const InstrItineraryData*,
+ const ScheduleDAG*) const;
};
} // End llvm namespace
Modified: llvm/trunk/include/llvm/Target/TargetInstrItineraries.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetInstrItineraries.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetInstrItineraries.h (original)
+++ llvm/trunk/include/llvm/Target/TargetInstrItineraries.h Thu Dec 23 23:03:26 2010
@@ -113,11 +113,13 @@
const unsigned *OperandCycles; ///< Array of operand cycles selected
const unsigned *Forwardings; ///< Array of pipeline forwarding pathes
const InstrItinerary *Itineraries; ///< Array of itineraries selected
+ unsigned IssueWidth; ///< Max issue per cycle. 0=Unknown.
/// Ctors.
///
InstrItineraryData() : Stages(0), OperandCycles(0), Forwardings(0),
- Itineraries(0) {}
+ Itineraries(0), IssueWidth(0) {}
+
InstrItineraryData(const InstrStage *S, const unsigned *OS,
const unsigned *F, const InstrItinerary *I)
: Stages(S), OperandCycles(OS), Forwardings(F), Itineraries(I) {}
Modified: llvm/trunk/lib/CodeGen/LatencyPriorityQueue.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LatencyPriorityQueue.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LatencyPriorityQueue.cpp (original)
+++ llvm/trunk/lib/CodeGen/LatencyPriorityQueue.cpp Thu Dec 23 23:03:26 2010
@@ -16,6 +16,7 @@
#define DEBUG_TYPE "scheduler"
#include "llvm/CodeGen/LatencyPriorityQueue.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
using namespace llvm;
bool latency_sort::operator()(const SUnit *LHS, const SUnit *RHS) const {
@@ -136,3 +137,16 @@
std::swap(*I, Queue.back());
Queue.pop_back();
}
+
+#ifdef NDEBUG
+void LatencyPriorityQueue::dump(ScheduleDAG *DAG) const {}
+#else
+void LatencyPriorityQueue::dump(ScheduleDAG *DAG) const {
+ LatencyPriorityQueue q = *this;
+ while (!q.empty()) {
+ SUnit *su = q.pop();
+ dbgs() << "Height " << su->getHeight() << ": ";
+ su->dump(DAG);
+ }
+}
+#endif
Modified: llvm/trunk/lib/CodeGen/PostRASchedulerList.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/PostRASchedulerList.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/PostRASchedulerList.cpp (original)
+++ llvm/trunk/lib/CodeGen/PostRASchedulerList.cpp Thu Dec 23 23:03:26 2010
@@ -133,18 +133,12 @@
std::vector<unsigned> KillIndices;
public:
- SchedulePostRATDList(MachineFunction &MF,
- const MachineLoopInfo &MLI,
- const MachineDominatorTree &MDT,
- ScheduleHazardRecognizer *HR,
- AntiDepBreaker *ADB,
- AliasAnalysis *aa)
- : ScheduleDAGInstrs(MF, MLI, MDT), Topo(SUnits),
- HazardRec(HR), AntiDepBreak(ADB), AA(aa),
- KillIndices(TRI->getNumRegs()) {}
+ SchedulePostRATDList(
+ MachineFunction &MF, MachineLoopInfo &MLI, MachineDominatorTree &MDT,
+ AliasAnalysis *AA, TargetSubtarget::AntiDepBreakMode AntiDepMode,
+ SmallVectorImpl<TargetRegisterClass*> &CriticalPathRCs);
- ~SchedulePostRATDList() {
- }
+ ~SchedulePostRATDList();
/// StartBlock - Initialize register live-range state for scheduling in
/// this block.
@@ -183,9 +177,34 @@
};
}
+SchedulePostRATDList::SchedulePostRATDList(
+ MachineFunction &MF, MachineLoopInfo &MLI, MachineDominatorTree &MDT,
+ AliasAnalysis *AA, TargetSubtarget::AntiDepBreakMode AntiDepMode,
+ SmallVectorImpl<TargetRegisterClass*> &CriticalPathRCs)
+ : ScheduleDAGInstrs(MF, MLI, MDT), Topo(SUnits), AA(AA),
+ KillIndices(TRI->getNumRegs())
+{
+ const TargetMachine &TM = MF.getTarget();
+ const InstrItineraryData *InstrItins = TM.getInstrItineraryData();
+ HazardRec =
+ TM.getInstrInfo()->CreateTargetPostRAHazardRecognizer(InstrItins, this);
+ AntiDepBreak =
+ ((AntiDepMode == TargetSubtarget::ANTIDEP_ALL) ?
+ (AntiDepBreaker *)new AggressiveAntiDepBreaker(MF, CriticalPathRCs) :
+ ((AntiDepMode == TargetSubtarget::ANTIDEP_CRITICAL) ?
+ (AntiDepBreaker *)new CriticalAntiDepBreaker(MF) : NULL));
+}
+
+SchedulePostRATDList::~SchedulePostRATDList() {
+ delete HazardRec;
+ delete AntiDepBreak;
+}
+
bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
- AA = &getAnalysis<AliasAnalysis>();
TII = Fn.getTarget().getInstrInfo();
+ MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
+ MachineDominatorTree &MDT = getAnalysis<MachineDominatorTree>();
+ AliasAnalysis *AA = &getAnalysis<AliasAnalysis>();
// Check for explicit enable/disable of post-ra scheduling.
TargetSubtarget::AntiDepBreakMode AntiDepMode = TargetSubtarget::ANTIDEP_NONE;
@@ -195,6 +214,7 @@
return false;
} else {
// Check that post-RA scheduling is enabled for this target.
+ // This may upgrade the AntiDepMode.
const TargetSubtarget &ST = Fn.getTarget().getSubtarget<TargetSubtarget>();
if (!ST.enablePostRAScheduler(OptLevel, AntiDepMode, CriticalPathRCs))
return false;
@@ -210,19 +230,8 @@
DEBUG(dbgs() << "PostRAScheduler\n");
- const MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
- const MachineDominatorTree &MDT = getAnalysis<MachineDominatorTree>();
- const TargetMachine &TM = Fn.getTarget();
- const InstrItineraryData *InstrItins = TM.getInstrItineraryData();
- ScheduleHazardRecognizer *HR =
- TM.getInstrInfo()->CreateTargetPostRAHazardRecognizer(InstrItins);
- AntiDepBreaker *ADB =
- ((AntiDepMode == TargetSubtarget::ANTIDEP_ALL) ?
- (AntiDepBreaker *)new AggressiveAntiDepBreaker(Fn, CriticalPathRCs) :
- ((AntiDepMode == TargetSubtarget::ANTIDEP_CRITICAL) ?
- (AntiDepBreaker *)new CriticalAntiDepBreaker(Fn) : NULL));
-
- SchedulePostRATDList Scheduler(Fn, MLI, MDT, HR, ADB, AA);
+ SchedulePostRATDList Scheduler(Fn, MLI, MDT, AA, AntiDepMode,
+ CriticalPathRCs);
// Loop over all of the basic blocks
for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
@@ -270,9 +279,6 @@
Scheduler.FixupKills(MBB);
}
- delete HR;
- delete ADB;
-
return true;
}
@@ -617,13 +623,7 @@
MinDepth = PendingQueue[i]->getDepth();
}
- DEBUG(dbgs() << "\n*** Examining Available\n";
- LatencyPriorityQueue q = AvailableQueue;
- while (!q.empty()) {
- SUnit *su = q.pop();
- dbgs() << "Height " << su->getHeight() << ": ";
- su->dump(this);
- });
+ DEBUG(dbgs() << "\n*** Examining Available\n"; AvailableQueue.dump(this));
SUnit *FoundSUnit = 0;
bool HasNoopHazards = false;
@@ -631,7 +631,7 @@
SUnit *CurSUnit = AvailableQueue.pop();
ScheduleHazardRecognizer::HazardType HT =
- HazardRec->getHazardType(CurSUnit);
+ HazardRec->getHazardType(CurSUnit, 0/*no stalls*/);
if (HT == ScheduleHazardRecognizer::NoHazard) {
FoundSUnit = CurSUnit;
break;
Modified: llvm/trunk/lib/CodeGen/ScheduleDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ScheduleDAG.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/ScheduleDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/ScheduleDAG.cpp Thu Dec 23 23:03:26 2010
@@ -15,6 +15,7 @@
#define DEBUG_TYPE "pre-RA-sched"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetRegisterInfo.h"
@@ -33,6 +34,12 @@
ScheduleDAG::~ScheduleDAG() {}
+/// getInstrDesc helper to handle SDNodes.
+const TargetInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const {
+ if (!Node->isMachineOpcode()) return NULL;
+ return &TII->get(Node->getMachineOpcode());
+}
+
/// dump - dump the schedule.
void ScheduleDAG::dumpSchedule() const {
for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
Modified: llvm/trunk/lib/CodeGen/ScoreboardHazardRecognizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ScoreboardHazardRecognizer.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/ScoreboardHazardRecognizer.cpp (original)
+++ llvm/trunk/lib/CodeGen/ScoreboardHazardRecognizer.cpp Thu Dec 23 23:03:26 2010
@@ -13,7 +13,7 @@
//
//===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "sched-hazard"
+#define DEBUG_TYPE ::llvm::ScoreboardHazardRecognizer::DebugType
#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/Support/Debug.h"
@@ -23,29 +23,48 @@
using namespace llvm;
+#ifndef NDEBUG
+const char *ScoreboardHazardRecognizer::DebugType = "";
+#endif
+
ScoreboardHazardRecognizer::
-ScoreboardHazardRecognizer(const InstrItineraryData *LItinData) :
- ScheduleHazardRecognizer(), ItinData(LItinData) {
+ScoreboardHazardRecognizer(const InstrItineraryData *II,
+ const ScheduleDAG *SchedDAG,
+ const char *ParentDebugType) :
+ ScheduleHazardRecognizer(), ItinData(II), DAG(SchedDAG), IssueWidth(0),
+ IssueCount(0) {
+
+#ifndef NDEBUG
+ DebugType = ParentDebugType;
+#endif
+
// Determine the maximum depth of any itinerary. This determines the
// depth of the scoreboard. We always make the scoreboard at least 1
// cycle deep to avoid dealing with the boundary condition.
unsigned ScoreboardDepth = 1;
if (ItinData && !ItinData->isEmpty()) {
+ IssueWidth = ItinData->IssueWidth;
+
for (unsigned idx = 0; ; ++idx) {
if (ItinData->isEndMarker(idx))
break;
const InstrStage *IS = ItinData->beginStage(idx);
const InstrStage *E = ItinData->endStage(idx);
+ unsigned CurCycle = 0;
unsigned ItinDepth = 0;
- for (; IS != E; ++IS)
- ItinDepth += IS->getCycles();
+ for (; IS != E; ++IS) {
+ unsigned StageDepth = CurCycle + IS->getCycles();
+ if (ItinDepth < StageDepth) ItinDepth = StageDepth;
+ CurCycle += IS->getNextCycles();
+ }
// Find the next power-of-2 >= ItinDepth
while (ItinDepth > ScoreboardDepth) {
ScoreboardDepth *= 2;
}
}
+ MaxLookAhead = ScoreboardDepth;
}
ReservedScoreboard.reset(ScoreboardDepth);
@@ -56,6 +75,7 @@
}
void ScoreboardHazardRecognizer::Reset() {
+ IssueCount = 0;
RequiredScoreboard.reset();
ReservedScoreboard.reset();
}
@@ -76,24 +96,46 @@
}
}
+bool ScoreboardHazardRecognizer::atIssueLimit() const {
+ if (IssueWidth == 0)
+ return false;
+
+ return IssueCount == IssueWidth;
+}
+
ScheduleHazardRecognizer::HazardType
-ScoreboardHazardRecognizer::getHazardType(SUnit *SU) {
+ScoreboardHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (!ItinData || ItinData->isEmpty())
return NoHazard;
- unsigned cycle = 0;
+ // Note that stalls will be negative for bottom-up scheduling.
+ int cycle = Stalls;
// Use the itinerary for the underlying instruction to check for
// free FU's in the scoreboard at the appropriate future cycles.
- unsigned idx = SU->getInstr()->getDesc().getSchedClass();
+
+ const TargetInstrDesc *TID = DAG->getInstrDesc(SU);
+ if (TID == NULL) {
+ // Don't check hazards for non-machineinstr Nodes.
+ return NoHazard;
+ }
+ unsigned idx = TID->getSchedClass();
for (const InstrStage *IS = ItinData->beginStage(idx),
*E = ItinData->endStage(idx); IS != E; ++IS) {
// We must find one of the stage's units free for every cycle the
// stage is occupied. FIXME it would be more accurate to find the
// same unit free in all the cycles.
for (unsigned int i = 0; i < IS->getCycles(); ++i) {
- assert(((cycle + i) < RequiredScoreboard.getDepth()) &&
- "Scoreboard depth exceeded!");
+ int StageCycle = cycle + (int)i;
+ if (StageCycle < 0)
+ continue;
+
+ if (StageCycle >= (int)RequiredScoreboard.getDepth()) {
+ assert((StageCycle - Stalls) < (int)RequiredScoreboard.getDepth() &&
+ "Scoreboard depth exceeded!");
+ // This stage was stalled beyond pipeline depth, so cannot conflict.
+ break;
+ }
unsigned freeUnits = IS->getUnits();
switch (IS->getReservationKind()) {
@@ -101,18 +143,18 @@
assert(0 && "Invalid FU reservation");
case InstrStage::Required:
// Required FUs conflict with both reserved and required ones
- freeUnits &= ~ReservedScoreboard[cycle + i];
+ freeUnits &= ~ReservedScoreboard[StageCycle];
// FALLTHROUGH
case InstrStage::Reserved:
// Reserved FUs can conflict only with required ones.
- freeUnits &= ~RequiredScoreboard[cycle + i];
+ freeUnits &= ~RequiredScoreboard[StageCycle];
break;
}
if (!freeUnits) {
DEBUG(dbgs() << "*** Hazard in cycle " << (cycle + i) << ", ");
DEBUG(dbgs() << "SU(" << SU->NodeNum << "): ");
- DEBUG(SU->getInstr()->dump());
+ DEBUG(DAG->dumpNode(SU));
return Hazard;
}
}
@@ -128,11 +170,15 @@
if (!ItinData || ItinData->isEmpty())
return;
+ ++IssueCount;
+
unsigned cycle = 0;
// Use the itinerary for the underlying instruction to reserve FU's
// in the scoreboard at the appropriate future cycles.
- unsigned idx = SU->getInstr()->getDesc().getSchedClass();
+ const TargetInstrDesc *TID = DAG->getInstrDesc(SU);
+ assert(TID && "The scheduler must filter non-machineinstrs");
+ unsigned idx = TID->getSchedClass();
for (const InstrStage *IS = ItinData->beginStage(idx),
*E = ItinData->endStage(idx); IS != E; ++IS) {
// We must reserve one of the stage's units for every cycle the
@@ -179,11 +225,13 @@
}
void ScoreboardHazardRecognizer::AdvanceCycle() {
+ IssueCount = 0;
ReservedScoreboard[0] = 0; ReservedScoreboard.advance();
RequiredScoreboard[0] = 0; RequiredScoreboard.advance();
}
void ScoreboardHazardRecognizer::RecedeCycle() {
+ IssueCount = 0;
ReservedScoreboard[ReservedScoreboard.getDepth()-1] = 0;
ReservedScoreboard.recede();
RequiredScoreboard[RequiredScoreboard.getDepth()-1] = 0;
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp Thu Dec 23 23:03:26 2010
@@ -63,11 +63,12 @@
public:
ScheduleDAGList(MachineFunction &mf,
- SchedulingPriorityQueue *availqueue,
- ScheduleHazardRecognizer *HR)
- : ScheduleDAGSDNodes(mf),
- AvailableQueue(availqueue), HazardRec(HR) {
- }
+ SchedulingPriorityQueue *availqueue)
+ : ScheduleDAGSDNodes(mf), AvailableQueue(availqueue) {
+
+ const TargetMachine &tm = mf.getTarget();
+ HazardRec = tm.getInstrInfo()->CreateTargetHazardRecognizer(&tm, this);
+ }
~ScheduleDAGList() {
delete HazardRec;
@@ -202,7 +203,7 @@
SUnit *CurSUnit = AvailableQueue->pop();
ScheduleHazardRecognizer::HazardType HT =
- HazardRec->getHazardType(CurSUnit);
+ HazardRec->getHazardType(CurSUnit, 0/*no stalls*/);
if (HT == ScheduleHazardRecognizer::NoHazard) {
FoundSUnit = CurSUnit;
break;
@@ -257,12 +258,8 @@
// Public Constructor Functions
//===----------------------------------------------------------------------===//
-/// createTDListDAGScheduler - This creates a top-down list scheduler with a
-/// new hazard recognizer. This scheduler takes ownership of the hazard
-/// recognizer and deletes it when done.
+/// createTDListDAGScheduler - This creates a top-down list scheduler.
ScheduleDAGSDNodes *
llvm::createTDListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
- return new ScheduleDAGList(*IS->MF,
- new LatencyPriorityQueue(),
- IS->CreateTargetHazardRecognizer());
+ return new ScheduleDAGList(*IS->MF, new LatencyPriorityQueue());
}
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp Thu Dec 23 23:03:26 2010
@@ -20,6 +20,7 @@
#include "llvm/InlineAsm.h"
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
#include "llvm/Target/TargetRegisterInfo.h"
#include "llvm/Target/TargetData.h"
#include "llvm/Target/TargetMachine.h"
@@ -65,6 +66,11 @@
"which tries to balance ILP and register pressure",
createILPListDAGScheduler);
+static cl::opt<bool> EnableSchedCycles(
+ "enable-sched-cycles",
+ cl::desc("Enable cycle-level precision during preRA scheduling"),
+ cl::init(false), cl::Hidden);
+
namespace {
//===----------------------------------------------------------------------===//
/// ScheduleDAGRRList - The actual register reduction list scheduler
@@ -83,9 +89,21 @@
/// AvailableQueue - The priority queue to use for the available SUnits.
SchedulingPriorityQueue *AvailableQueue;
+ /// PendingQueue - This contains all of the instructions whose operands have
+ /// been issued, but their results are not ready yet (due to the latency of
+ /// the operation). Once the operands becomes available, the instruction is
+ /// added to the AvailableQueue.
+ std::vector<SUnit*> PendingQueue;
+
+ /// HazardRec - The hazard recognizer to use.
+ ScheduleHazardRecognizer *HazardRec;
+
/// CurCycle - The current scheduler state corresponds to this cycle.
unsigned CurCycle;
+ /// MinAvailableCycle - Cycle of the soonest available instruction.
+ unsigned MinAvailableCycle;
+
/// LiveRegDefs - A set of physical registers and their definition
/// that are "live". These nodes must be scheduled before any other nodes that
/// modifies the registers can be scheduled.
@@ -98,14 +116,22 @@
ScheduleDAGTopologicalSort Topo;
public:
- ScheduleDAGRRList(MachineFunction &mf,
- bool isbottomup, bool needlatency,
- SchedulingPriorityQueue *availqueue)
- : ScheduleDAGSDNodes(mf), isBottomUp(isbottomup), NeedLatency(needlatency),
- AvailableQueue(availqueue), CurCycle(0), Topo(SUnits) {
- }
+ ScheduleDAGRRList(MachineFunction &mf, bool needlatency,
+ SchedulingPriorityQueue *availqueue,
+ CodeGenOpt::Level OptLevel)
+ : ScheduleDAGSDNodes(mf), isBottomUp(availqueue->isBottomUp()),
+ NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0),
+ Topo(SUnits) {
+
+ const TargetMachine &tm = mf.getTarget();
+ if (EnableSchedCycles && OptLevel != CodeGenOpt::None)
+ HazardRec = tm.getInstrInfo()->CreateTargetHazardRecognizer(&tm, this);
+ else
+ HazardRec = new ScheduleHazardRecognizer();
+ }
~ScheduleDAGRRList() {
+ delete HazardRec;
delete AvailableQueue;
}
@@ -139,20 +165,31 @@
}
private:
+ bool isReady(SUnit *SU) {
+ return !EnableSchedCycles || !AvailableQueue->hasReadyFilter() ||
+ AvailableQueue->isReady(SU);
+ }
+
void ReleasePred(SUnit *SU, const SDep *PredEdge);
void ReleasePredecessors(SUnit *SU);
void ReleaseSucc(SUnit *SU, const SDep *SuccEdge);
void ReleaseSuccessors(SUnit *SU);
- void CapturePred(SDep *PredEdge);
+ void ReleasePending();
+ void AdvanceToCycle(unsigned NextCycle);
+ void AdvancePastStalls(SUnit *SU);
+ void EmitNode(SUnit *SU);
void ScheduleNodeBottomUp(SUnit*);
+ void CapturePred(SDep *PredEdge);
void UnscheduleNodeBottomUp(SUnit*);
- void BacktrackBottomUp(SUnit*, unsigned);
+ void RestoreHazardCheckerBottomUp();
+ void BacktrackBottomUp(SUnit*, SUnit*);
SUnit *CopyAndMoveSuccessors(SUnit*);
void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
const TargetRegisterClass*,
const TargetRegisterClass*,
SmallVector<SUnit*, 2>&);
bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&);
+
SUnit *PickNodeToScheduleBottomUp();
void ListScheduleBottomUp();
@@ -198,6 +235,7 @@
<< " '" << BB->getName() << "' **********\n");
CurCycle = 0;
+ MinAvailableCycle = EnableSchedCycles ? UINT_MAX : 0;
NumLiveRegs = 0;
LiveRegDefs.resize(TRI->getNumRegs(), NULL);
LiveRegGens.resize(TRI->getNumRegs(), NULL);
@@ -211,6 +249,8 @@
AvailableQueue->initNodes(SUnits);
+ HazardRec->Reset();
+
// Execute the actual scheduling loop Top-Down or Bottom-Up as appropriate.
if (isBottomUp)
ListScheduleBottomUp();
@@ -249,7 +289,20 @@
// to be scheduled. Ignore the special EntrySU node.
if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
PredSU->isAvailable = true;
- AvailableQueue->push(PredSU);
+
+ unsigned Height = PredSU->getHeight();
+ if (Height < MinAvailableCycle)
+ MinAvailableCycle = Height;
+
+ if (isReady(SU)) {
+ AvailableQueue->push(PredSU);
+ }
+ // CapturePred and others may have left the node in the pending queue, avoid
+ // adding it twice.
+ else if (!PredSU->isPending) {
+ PredSU->isPending = true;
+ PendingQueue.push_back(PredSU);
+ }
}
}
@@ -292,6 +345,127 @@
}
}
+/// Check to see if any of the pending instructions are ready to issue. If
+/// so, add them to the available queue.
+void ScheduleDAGRRList::ReleasePending() {
+ assert(!EnableSchedCycles && "requires --enable-sched-cycles" );
+
+ // If the available queue is empty, it is safe to reset MinAvailableCycle.
+ if (AvailableQueue->empty())
+ MinAvailableCycle = UINT_MAX;
+
+ // Check to see if any of the pending instructions are ready to issue. If
+ // so, add them to the available queue.
+ for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) {
+ unsigned ReadyCycle =
+ isBottomUp ? PendingQueue[i]->getHeight() : PendingQueue[i]->getDepth();
+ if (ReadyCycle < MinAvailableCycle)
+ MinAvailableCycle = ReadyCycle;
+
+ if (PendingQueue[i]->isAvailable) {
+ if (!isReady(PendingQueue[i]))
+ continue;
+ AvailableQueue->push(PendingQueue[i]);
+ }
+ PendingQueue[i]->isPending = false;
+ PendingQueue[i] = PendingQueue.back();
+ PendingQueue.pop_back();
+ --i; --e;
+ }
+}
+
+/// Move the scheduler state forward by the specified number of Cycles.
+void ScheduleDAGRRList::AdvanceToCycle(unsigned NextCycle) {
+ if (NextCycle <= CurCycle)
+ return;
+
+ AvailableQueue->setCurCycle(NextCycle);
+ if (HazardRec->getMaxLookAhead() == 0) {
+ // Bypass lots of virtual calls in case of long latency.
+ CurCycle = NextCycle;
+ }
+ else {
+ for (; CurCycle != NextCycle; ++CurCycle) {
+ if (isBottomUp)
+ HazardRec->RecedeCycle();
+ else
+ HazardRec->AdvanceCycle();
+ }
+ }
+ // FIXME: Instead of visiting the pending Q each time, set a dirty flag on the
+ // available Q to release pending nodes at least once before popping.
+ ReleasePending();
+}
+
+/// Move the scheduler state forward until the specified node's dependents are
+/// ready and can be scheduled with no resource conflicts.
+void ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) {
+ if (!EnableSchedCycles)
+ return;
+
+ unsigned ReadyCycle = isBottomUp ? SU->getHeight() : SU->getDepth();
+
+ // Bump CurCycle to account for latency. We assume the latency of other
+ // available instructions may be hidden by the stall (not a full pipe stall).
+ // This updates the hazard recognizer's cycle before reserving resources for
+ // this instruction.
+ AdvanceToCycle(ReadyCycle);
+
+ // Calls are scheduled in their preceding cycle, so don't conflict with
+ // hazards from instructions after the call. EmitNode will reset the
+ // scoreboard state before emitting the call.
+ if (isBottomUp && SU->isCall)
+ return;
+
+ // FIXME: For resource conflicts in very long non-pipelined stages, we
+ // should probably skip ahead here to avoid useless scoreboard checks.
+ int Stalls = 0;
+ while (true) {
+ ScheduleHazardRecognizer::HazardType HT =
+ HazardRec->getHazardType(SU, isBottomUp ? -Stalls : Stalls);
+
+ if (HT == ScheduleHazardRecognizer::NoHazard)
+ break;
+
+ ++Stalls;
+ }
+ AdvanceToCycle(CurCycle + Stalls);
+}
+
+/// Record this SUnit in the HazardRecognizer.
+/// Does not update CurCycle.
+void ScheduleDAGRRList::EmitNode(SUnit *SU) {
+ switch (SU->getNode()->getOpcode()) {
+ default:
+ assert(SU->getNode()->isMachineOpcode() &&
+ "This target-independent node should not be scheduled.");
+ break;
+ case ISD::MERGE_VALUES:
+ case ISD::TokenFactor:
+ case ISD::CopyToReg:
+ case ISD::CopyFromReg:
+ case ISD::EH_LABEL:
+ // Noops don't affect the scoreboard state. Copies are likely to be
+ // removed.
+ return;
+ case ISD::INLINEASM:
+ // For inline asm, clear the pipeline state.
+ HazardRec->Reset();
+ return;
+ }
+ if (isBottomUp && SU->isCall) {
+ // Calls are scheduled with their preceding instructions. For bottom-up
+ // scheduling, clear the pipeline state before emitting.
+ HazardRec->Reset();
+ }
+
+ HazardRec->EmitInstruction(SU);
+
+ if (!isBottomUp && SU->isCall) {
+ HazardRec->Reset();
+ }
+}
+
/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
/// count of its predecessors. If a predecessor pending count is zero, add it to
/// the Available queue.
@@ -304,8 +478,15 @@
DEBUG(dbgs() << " Height [" << SU->getHeight() << "] pipeline stall!\n");
#endif
- // FIXME: Handle noop hazard.
+ // FIXME: Do not modify node height. It may interfere with
+ // backtracking. Instead add a "ready cycle" to SUnit. Before scheduling the
+ // node it's ready cycle can aid heuristics, and after scheduling it can
+ // indicate the scheduled cycle.
SU->setHeightToAtLeast(CurCycle);
+
+ // Reserve resources for the scheduled intruction.
+ EmitNode(SU);
+
Sequence.push_back(SU);
AvailableQueue->ScheduledNode(SU);
@@ -327,6 +508,15 @@
}
SU->isScheduled = true;
+
+ // Conditions under which the scheduler should eagerly advance the cycle:
+ // (1) No available instructions
+ // (2) All pipelines full, so available instructions must have hazards.
+ //
+ // If SchedCycles is disabled, count each inst as one cycle.
+ if (!EnableSchedCycles ||
+ AvailableQueue->empty() || HazardRec->atIssueLimit())
+ AdvanceToCycle(CurCycle + 1);
}
/// CapturePred - This does the opposite of ReleasePred. Since SU is being
@@ -377,31 +567,69 @@
LiveRegGens[I->getReg()] = I->getSUnit();
}
}
+ if (SU->getHeight() < MinAvailableCycle)
+ MinAvailableCycle = SU->getHeight();
SU->setHeightDirty();
SU->isScheduled = false;
SU->isAvailable = true;
- AvailableQueue->push(SU);
+ if (EnableSchedCycles && AvailableQueue->hasReadyFilter()) {
+ // Don't make available until backtracking is complete.
+ SU->isPending = true;
+ PendingQueue.push_back(SU);
+ }
+ else {
+ AvailableQueue->push(SU);
+ }
AvailableQueue->UnscheduledNode(SU);
}
+/// After backtracking, the hazard checker needs to be restored to a state
+/// corresponding the the current cycle.
+void ScheduleDAGRRList::RestoreHazardCheckerBottomUp() {
+ HazardRec->Reset();
+
+ unsigned LookAhead = std::min((unsigned)Sequence.size(),
+ HazardRec->getMaxLookAhead());
+ if (LookAhead == 0)
+ return;
+
+ std::vector<SUnit*>::const_iterator I = (Sequence.end() - LookAhead);
+ unsigned HazardCycle = (*I)->getHeight();
+ for (std::vector<SUnit*>::const_iterator E = Sequence.end(); I != E; ++I) {
+ SUnit *SU = *I;
+ for (; SU->getHeight() > HazardCycle; ++HazardCycle) {
+ HazardRec->RecedeCycle();
+ }
+ EmitNode(SU);
+ }
+}
+
/// BacktrackBottomUp - Backtrack scheduling to a previous cycle specified in
/// BTCycle in order to schedule a specific node.
-void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, unsigned BtCycle) {
- SUnit *OldSU = NULL;
- while (CurCycle > BtCycle) {
- OldSU = Sequence.back();
+void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) {
+ SUnit *OldSU = Sequence.back();
+ while (true) {
Sequence.pop_back();
if (SU->isSucc(OldSU))
// Don't try to remove SU from AvailableQueue.
SU->isAvailable = false;
+ // FIXME: use ready cycle instead of height
+ CurCycle = OldSU->getHeight();
UnscheduleNodeBottomUp(OldSU);
- --CurCycle;
AvailableQueue->setCurCycle(CurCycle);
+ if (OldSU == BtSU)
+ break;
+ OldSU = Sequence.back();
}
assert(!SU->isSucc(OldSU) && "Something is wrong!");
+ RestoreHazardCheckerBottomUp();
+
+ if (EnableSchedCycles)
+ ReleasePending();
+
++NumBacktracks;
}
@@ -741,7 +969,7 @@
/// Return a node that can be scheduled in this cycle. Requirements:
/// (1) Ready: latency has been satisfied
-/// (2) No Hazards: resources are available (TBD)
+/// (2) No Hazards: resources are available
/// (3) No Interferences: may unschedule to break register interferences.
SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() {
SmallVector<SUnit*, 4> Interferences;
@@ -777,24 +1005,26 @@
// Try unscheduling up to the point where it's safe to schedule
// this node.
- unsigned LiveCycle = CurCycle;
+ SUnit *BtSU = NULL;
+ unsigned LiveCycle = UINT_MAX;
for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) {
unsigned Reg = LRegs[j];
- unsigned LCycle = LiveRegGens[Reg]->getHeight();
- LiveCycle = std::min(LiveCycle, LCycle);
+ if (LiveRegGens[Reg]->getHeight() < LiveCycle) {
+ BtSU = LiveRegGens[Reg];
+ LiveCycle = BtSU->getHeight();
+ }
}
- SUnit *OldSU = Sequence[LiveCycle];
- if (!WillCreateCycle(TrySU, OldSU)) {
- BacktrackBottomUp(TrySU, LiveCycle);
+ if (!WillCreateCycle(TrySU, BtSU)) {
+ BacktrackBottomUp(TrySU, BtSU);
// Force the current node to be scheduled before the node that
// requires the physical reg dep.
- if (OldSU->isAvailable) {
- OldSU->isAvailable = false;
- if (!OldSU->isPending)
- AvailableQueue->remove(OldSU);
+ if (BtSU->isAvailable) {
+ BtSU->isAvailable = false;
+ if (!BtSU->isPending)
+ AvailableQueue->remove(BtSU);
}
- AddPred(TrySU, SDep(OldSU, SDep::Order, /*Latency=*/1,
+ AddPred(TrySU, SDep(BtSU, SDep::Order, /*Latency=*/1,
/*Reg=*/0, /*isNormalMemory=*/false,
/*isMustAlias=*/false, /*isArtificial=*/true));
@@ -891,15 +1121,22 @@
// priority. If it is not ready put it back. Schedule the node.
Sequence.reserve(SUnits.size());
while (!AvailableQueue->empty()) {
+ DEBUG(dbgs() << "\n*** Examining Available\n";
+ AvailableQueue->dump(this));
+
// Pick the best node to schedule taking all constraints into
// consideration.
SUnit *SU = PickNodeToScheduleBottomUp();
- if (SU)
- ScheduleNodeBottomUp(SU);
+ AdvancePastStalls(SU);
- ++CurCycle;
- AvailableQueue->setCurCycle(CurCycle);
+ ScheduleNodeBottomUp(SU);
+
+ while (AvailableQueue->empty() && !PendingQueue.empty()) {
+ // Advance the cycle to free resources. Skip ahead to the next ready SU.
+ assert(MinAvailableCycle < UINT_MAX && "MinAvailableCycle uninitialized");
+ AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle));
+ }
}
// Reverse the order if it is bottom up.
@@ -967,7 +1204,6 @@
/// ListScheduleTopDown - The main loop of list scheduling for top-down
/// schedulers.
void ScheduleDAGRRList::ListScheduleTopDown() {
- unsigned CurCycle = 0;
AvailableQueue->setCurCycle(CurCycle);
// Release any successors of the special Entry node.
@@ -1011,9 +1247,18 @@
template<class SF>
class RegReductionPriorityQueue;
+ struct queue_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ bool isReady(SUnit* SU, unsigned CurCycle) const { return true; }
+ };
+
/// bu_ls_rr_sort - Priority function for bottom up register pressure
// reduction scheduler.
- struct bu_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ struct bu_ls_rr_sort : public queue_sort {
+ enum {
+ IsBottomUp = true,
+ HasReadyFilter = false
+ };
+
RegReductionPriorityQueue<bu_ls_rr_sort> *SPQ;
bu_ls_rr_sort(RegReductionPriorityQueue<bu_ls_rr_sort> *spq) : SPQ(spq) {}
bu_ls_rr_sort(const bu_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {}
@@ -1023,8 +1268,13 @@
// td_ls_rr_sort - Priority function for top down register pressure reduction
// scheduler.
- struct td_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
- RegReductionPriorityQueue<td_ls_rr_sort> *SPQ;
+ struct td_ls_rr_sort : public queue_sort {
+ enum {
+ IsBottomUp = false,
+ HasReadyFilter = false
+ };
+
+ RegReductionPriorityQueue<td_ls_rr_sort> *SPQ;
td_ls_rr_sort(RegReductionPriorityQueue<td_ls_rr_sort> *spq) : SPQ(spq) {}
td_ls_rr_sort(const td_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {}
@@ -1032,7 +1282,12 @@
};
// src_ls_rr_sort - Priority function for source order scheduler.
- struct src_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ struct src_ls_rr_sort : public queue_sort {
+ enum {
+ IsBottomUp = true,
+ HasReadyFilter = false
+ };
+
RegReductionPriorityQueue<src_ls_rr_sort> *SPQ;
src_ls_rr_sort(RegReductionPriorityQueue<src_ls_rr_sort> *spq)
: SPQ(spq) {}
@@ -1043,7 +1298,12 @@
};
// hybrid_ls_rr_sort - Priority function for hybrid scheduler.
- struct hybrid_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ struct hybrid_ls_rr_sort : public queue_sort {
+ enum {
+ IsBottomUp = true,
+ HasReadyFilter = false
+ };
+
RegReductionPriorityQueue<hybrid_ls_rr_sort> *SPQ;
hybrid_ls_rr_sort(RegReductionPriorityQueue<hybrid_ls_rr_sort> *spq)
: SPQ(spq) {}
@@ -1055,13 +1315,20 @@
// ilp_ls_rr_sort - Priority function for ILP (instruction level parallelism)
// scheduler.
- struct ilp_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ struct ilp_ls_rr_sort : public queue_sort {
+ enum {
+ IsBottomUp = true,
+ HasReadyFilter = true
+ };
+
RegReductionPriorityQueue<ilp_ls_rr_sort> *SPQ;
ilp_ls_rr_sort(RegReductionPriorityQueue<ilp_ls_rr_sort> *spq)
: SPQ(spq) {}
ilp_ls_rr_sort(const ilp_ls_rr_sort &RHS)
: SPQ(RHS.SPQ) {}
+ bool isReady(SUnit *SU, unsigned CurCycle) const;
+
bool operator()(const SUnit* left, const SUnit* right) const;
};
} // end anonymous namespace
@@ -1098,6 +1365,19 @@
namespace {
template<class SF>
class RegReductionPriorityQueue : public SchedulingPriorityQueue {
+ static SUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker) {
+ std::vector<SUnit *>::iterator Best = Q.begin();
+ for (std::vector<SUnit *>::iterator I = llvm::next(Q.begin()),
+ E = Q.end(); I != E; ++I)
+ if (Picker(*Best, *I))
+ Best = I;
+ SUnit *V = *Best;
+ if (Best != prior(Q.end()))
+ std::swap(*Best, Q.back());
+ Q.pop_back();
+ return V;
+ }
+
std::vector<SUnit*> Queue;
SF Picker;
unsigned CurQueueId;
@@ -1130,7 +1410,8 @@
const TargetInstrInfo *tii,
const TargetRegisterInfo *tri,
const TargetLowering *tli)
- : Picker(this), CurQueueId(0), TracksRegPressure(tracksrp),
+ : SchedulingPriorityQueue(SF::HasReadyFilter), Picker(this),
+ CurQueueId(0), TracksRegPressure(tracksrp),
MF(mf), TII(tii), TRI(tri), TLI(tli), scheduleDAG(NULL) {
if (TracksRegPressure) {
unsigned NumRC = TRI->getNumRegClasses();
@@ -1144,6 +1425,8 @@
}
}
+ bool isBottomUp() const { return SF::IsBottomUp; }
+
void initNodes(std::vector<SUnit> &sunits) {
SUnits = &sunits;
// Add pseudo dependency edges for two-address nodes.
@@ -1205,6 +1488,10 @@
bool empty() const { return Queue.empty(); }
+ bool isReady(SUnit *U) const {
+ return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle());
+ }
+
void push(SUnit *U) {
assert(!U->NodeQueueId && "Node in the queue already");
U->NodeQueueId = ++CurQueueId;
@@ -1212,16 +1499,9 @@
}
SUnit *pop() {
- if (empty()) return NULL;
- std::vector<SUnit *>::iterator Best = Queue.begin();
- for (std::vector<SUnit *>::iterator I = llvm::next(Queue.begin()),
- E = Queue.end(); I != E; ++I)
- if (Picker(*Best, *I))
- Best = I;
- SUnit *V = *Best;
- if (Best != prior(Queue.end()))
- std::swap(*Best, Queue.back());
- Queue.pop_back();
+ if (Queue.empty()) return NULL;
+
+ SUnit *V = popFromQueue(Queue, Picker);
V->NodeQueueId = 0;
return V;
}
@@ -1475,6 +1755,20 @@
}
}
+ void dump(ScheduleDAG *DAG) const {
+ // Emulate pop() without clobbering NodeQueueIds.
+ std::vector<SUnit*> DumpQueue = Queue;
+ SF DumpPicker = Picker;
+ while (!DumpQueue.empty()) {
+ SUnit *SU = popFromQueue(DumpQueue, DumpPicker);
+ if (isBottomUp())
+ dbgs() << "Height " << SU->getHeight() << ": ";
+ else
+ dbgs() << "Depth " << SU->getDepth() << ": ";
+ SU->dump(DAG);
+ }
+ }
+
protected:
bool canClobber(const SUnit *SU, const SUnit *Op);
void AddPseudoTwoAddrDeps();
@@ -1605,6 +1899,7 @@
if (LScratch != RScratch)
return LScratch > RScratch;
+ // Note: with a bottom-up ready filter, the height check may be redundant.
if (left->getHeight() != right->getHeight())
return left->getHeight() > right->getHeight();
@@ -1696,6 +1991,12 @@
return BURRSort(left, right, SPQ);
}
+// Schedule as many instructions in each cycle as possible. So don't make an
+// instruction available unless it is ready in the current cycle.
+bool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
+ return SU->getHeight() <= CurCycle;
+}
+
bool ilp_ls_rr_sort::operator()(const SUnit *left,
const SUnit *right) const {
if (left->isCall || right->isCall)
@@ -2051,46 +2352,50 @@
//===----------------------------------------------------------------------===//
llvm::ScheduleDAGSDNodes *
-llvm::createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
+llvm::createBURRListDAGScheduler(SelectionDAGISel *IS,
+ CodeGenOpt::Level OptLevel) {
const TargetMachine &TM = IS->TM;
const TargetInstrInfo *TII = TM.getInstrInfo();
const TargetRegisterInfo *TRI = TM.getRegisterInfo();
BURegReductionPriorityQueue *PQ =
new BURegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0);
- ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, false, PQ);
+ ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel);
PQ->setScheduleDAG(SD);
return SD;
}
llvm::ScheduleDAGSDNodes *
-llvm::createTDRRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
+llvm::createTDRRListDAGScheduler(SelectionDAGISel *IS,
+ CodeGenOpt::Level OptLevel) {
const TargetMachine &TM = IS->TM;
const TargetInstrInfo *TII = TM.getInstrInfo();
const TargetRegisterInfo *TRI = TM.getRegisterInfo();
TDRegReductionPriorityQueue *PQ =
new TDRegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0);
- ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, false, PQ);
+ ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel);
PQ->setScheduleDAG(SD);
return SD;
}
llvm::ScheduleDAGSDNodes *
-llvm::createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
+llvm::createSourceListDAGScheduler(SelectionDAGISel *IS,
+ CodeGenOpt::Level OptLevel) {
const TargetMachine &TM = IS->TM;
const TargetInstrInfo *TII = TM.getInstrInfo();
const TargetRegisterInfo *TRI = TM.getRegisterInfo();
SrcRegReductionPriorityQueue *PQ =
new SrcRegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0);
- ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, false, PQ);
+ ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel);
PQ->setScheduleDAG(SD);
return SD;
}
llvm::ScheduleDAGSDNodes *
-llvm::createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
+llvm::createHybridListDAGScheduler(SelectionDAGISel *IS,
+ CodeGenOpt::Level OptLevel) {
const TargetMachine &TM = IS->TM;
const TargetInstrInfo *TII = TM.getInstrInfo();
const TargetRegisterInfo *TRI = TM.getRegisterInfo();
@@ -2098,13 +2403,15 @@
HybridBURRPriorityQueue *PQ =
new HybridBURRPriorityQueue(*IS->MF, true, TII, TRI, TLI);
- ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, true, PQ);
+
+ ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel);
PQ->setScheduleDAG(SD);
return SD;
}
llvm::ScheduleDAGSDNodes *
-llvm::createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
+llvm::createILPListDAGScheduler(SelectionDAGISel *IS,
+ CodeGenOpt::Level OptLevel) {
const TargetMachine &TM = IS->TM;
const TargetInstrInfo *TII = TM.getInstrInfo();
const TargetRegisterInfo *TRI = TM.getRegisterInfo();
@@ -2112,7 +2419,7 @@
ILPBURRPriorityQueue *PQ =
new ILPBURRPriorityQueue(*IS->MF, true, TII, TRI, TLI);
- ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, true, PQ);
+ ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel);
PQ->setScheduleDAG(SD);
return SD;
}
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp Thu Dec 23 23:03:26 2010
@@ -1201,10 +1201,6 @@
return Ctor(this, OptLevel);
}
-ScheduleHazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() {
- return new ScheduleHazardRecognizer();
-}
-
//===----------------------------------------------------------------------===//
// Helper functions used by the generated instruction selector.
//===----------------------------------------------------------------------===//
Modified: llvm/trunk/lib/CodeGen/TargetInstrInfoImpl.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TargetInstrInfoImpl.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TargetInstrInfoImpl.cpp (original)
+++ llvm/trunk/lib/CodeGen/TargetInstrInfoImpl.cpp Thu Dec 23 23:03:26 2010
@@ -414,8 +414,18 @@
return false;
}
+// Default implementation of CreateTargetPreRAHazardRecognizer.
+ScheduleHazardRecognizer *TargetInstrInfoImpl::
+CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const {
+ // Dummy hazard recognizer allows all instructions to issue.
+ return new ScheduleHazardRecognizer();
+}
+
// Default implementation of CreateTargetPostRAHazardRecognizer.
ScheduleHazardRecognizer *TargetInstrInfoImpl::
-CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II) const {
- return (ScheduleHazardRecognizer *)new ScoreboardHazardRecognizer(II);
+CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
+ const ScheduleDAG *DAG) const {
+ return (ScheduleHazardRecognizer *)
+ new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
}
Modified: llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.cpp (original)
+++ llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.cpp Thu Dec 23 23:03:26 2010
@@ -41,6 +41,13 @@
EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
cl::desc("Enable ARM 2-addr to 3-addr conv"));
+// Other targets already have a hazard recognizer enabled by default, so this
+// flag currently only affects ARM. It will be generalized when it becomes a
+// disabled flag.
+static cl::opt<bool> EnableHazardRecognizer(
+ "enable-sched-hazard", cl::Hidden,
+ cl::desc("Enable hazard detection during preRA scheduling"),
+ cl::init(false));
/// ARM_MLxEntry - Record information about MLA / MLS instructions.
struct ARM_MLxEntry {
@@ -85,12 +92,25 @@
}
}
+// Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
+// currently defaults to no prepass hazard recognizer.
ScheduleHazardRecognizer *ARMBaseInstrInfo::
-CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II) const {
+CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const {
+ if (EnableHazardRecognizer) {
+ const InstrItineraryData *II = TM->getInstrItineraryData();
+ return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
+ }
+ return TargetInstrInfoImpl::CreateTargetHazardRecognizer(TM, DAG);
+}
+
+ScheduleHazardRecognizer *ARMBaseInstrInfo::
+CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
+ const ScheduleDAG *DAG) const {
if (Subtarget.isThumb2() || Subtarget.hasVFP2())
return (ScheduleHazardRecognizer *)
- new ARMHazardRecognizer(II, *this, getRegisterInfo(), Subtarget);
- return TargetInstrInfoImpl::CreateTargetPostRAHazardRecognizer(II);
+ new ARMHazardRecognizer(II, *this, getRegisterInfo(), Subtarget, DAG);
+ return TargetInstrInfoImpl::CreateTargetPostRAHazardRecognizer(II, DAG);
}
MachineInstr *
Modified: llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.h (original)
+++ llvm/trunk/lib/Target/ARM/ARMBaseInstrInfo.h Thu Dec 23 23:03:26 2010
@@ -211,7 +211,12 @@
const ARMSubtarget &getSubtarget() const { return Subtarget; }
ScheduleHazardRecognizer *
- CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II) const;
+ CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const;
+
+ ScheduleHazardRecognizer *
+ CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
+ const ScheduleDAG *DAG) const;
// Branch analysis.
virtual bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
Modified: llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.cpp (original)
+++ llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.cpp Thu Dec 23 23:03:26 2010
@@ -34,7 +34,9 @@
}
ScheduleHazardRecognizer::HazardType
-ARMHazardRecognizer::getHazardType(SUnit *SU) {
+ARMHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
+ assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead");
+
MachineInstr *MI = SU->getInstr();
if (!MI->isDebugValue()) {
@@ -61,19 +63,19 @@
(TII.canCauseFpMLxStall(MI->getOpcode()) ||
hasRAWHazard(DefMI, MI, TRI))) {
// Try to schedule another instruction for the next 4 cycles.
- if (Stalls == 0)
- Stalls = 4;
+ if (FpMLxStalls == 0)
+ FpMLxStalls = 4;
return Hazard;
}
}
}
- return ScoreboardHazardRecognizer::getHazardType(SU);
+ return ScoreboardHazardRecognizer::getHazardType(SU, Stalls);
}
void ARMHazardRecognizer::Reset() {
LastMI = 0;
- Stalls = 0;
+ FpMLxStalls = 0;
ITBlockSize = 0;
ScoreboardHazardRecognizer::Reset();
}
@@ -100,14 +102,14 @@
if (!MI->isDebugValue()) {
LastMI = MI;
- Stalls = 0;
+ FpMLxStalls = 0;
}
ScoreboardHazardRecognizer::EmitInstruction(SU);
}
void ARMHazardRecognizer::AdvanceCycle() {
- if (Stalls && --Stalls == 0)
+ if (FpMLxStalls && --FpMLxStalls == 0)
// Stalled for 4 cycles but still can't schedule any other instructions.
LastMI = 0;
ScoreboardHazardRecognizer::AdvanceCycle();
Modified: llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.h (original)
+++ llvm/trunk/lib/Target/ARM/ARMHazardRecognizer.h Thu Dec 23 23:03:26 2010
@@ -29,7 +29,7 @@
const ARMSubtarget &STI;
MachineInstr *LastMI;
- unsigned Stalls;
+ unsigned FpMLxStalls;
unsigned ITBlockSize; // No. of MIs in current IT block yet to be scheduled.
MachineInstr *ITBlockMIs[4];
@@ -37,11 +37,12 @@
ARMHazardRecognizer(const InstrItineraryData *ItinData,
const ARMBaseInstrInfo &tii,
const ARMBaseRegisterInfo &tri,
- const ARMSubtarget &sti) :
- ScoreboardHazardRecognizer(ItinData), TII(tii), TRI(tri), STI(sti),
- LastMI(0), ITBlockSize(0) {}
+ const ARMSubtarget &sti,
+ const ScheduleDAG *DAG) :
+ ScoreboardHazardRecognizer(ItinData, DAG, "post-RA-sched"), TII(tii),
+ TRI(tri), STI(sti), LastMI(0), ITBlockSize(0) {}
- virtual HazardType getHazardType(SUnit *SU);
+ virtual HazardType getHazardType(SUnit *SU, int Stalls);
virtual void Reset();
virtual void EmitInstruction(SUnit *SU);
virtual void AdvanceCycle();
Modified: llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp (original)
+++ llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp Thu Dec 23 23:03:26 2010
@@ -140,6 +140,9 @@
FSWithArch = FS;
CPUString = ParseSubtargetFeatures(FSWithArch, CPUString);
+ // After parsing Itineraries, set ItinData.IssueWidth.
+ computeIssueWidth();
+
// Thumb2 implies at least V6T2.
if (ARMArchVersion >= V6T2)
ThumbMode = Thumb2;
@@ -224,6 +227,21 @@
return 10;
}
+void ARMSubtarget::computeIssueWidth() {
+ unsigned allStage1Units = 0;
+ for (const InstrItinerary *itin = InstrItins.Itineraries;
+ itin->FirstStage != ~0U; ++itin) {
+ const InstrStage *IS = InstrItins.Stages + itin->FirstStage;
+ allStage1Units |= IS->getUnits();
+ }
+ InstrItins.IssueWidth = 0;
+ while (allStage1Units) {
+ ++InstrItins.IssueWidth;
+ // clear the lowest bit
+ allStage1Units ^= allStage1Units & ~(allStage1Units - 1);
+ }
+}
+
bool ARMSubtarget::enablePostRAScheduler(
CodeGenOpt::Level OptLevel,
TargetSubtarget::AntiDepBreakMode& Mode,
Modified: llvm/trunk/lib/Target/ARM/ARMSubtarget.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMSubtarget.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMSubtarget.h (original)
+++ llvm/trunk/lib/Target/ARM/ARMSubtarget.h Thu Dec 23 23:03:26 2010
@@ -156,6 +156,8 @@
std::string ParseSubtargetFeatures(const std::string &FS,
const std::string &CPU);
+ void computeIssueWidth();
+
bool hasV4TOps() const { return ARMArchVersion >= V4T; }
bool hasV5TOps() const { return ARMArchVersion >= V5T; }
bool hasV5TEOps() const { return ARMArchVersion >= V5TE; }
Modified: llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.cpp (original)
+++ llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.cpp Thu Dec 23 23:03:26 2010
@@ -41,12 +41,14 @@
///
/// \return NoHazard
ScheduleHazardRecognizer::HazardType
-SPUHazardRecognizer::getHazardType(SUnit *SU)
+SPUHazardRecognizer::getHazardType(SUnit *SU, int Stalls)
{
// Initial thoughts on how to do this, but this code cannot work unless the
// function's prolog and epilog code are also being scheduled so that we can
// accurately determine which pipeline is being scheduled.
#if 0
+ assert(Stalls == 0 && "SPU hazards don't yet support scoreboard lookahead");
+
const SDNode *Node = SU->getNode()->getFlaggedMachineNode();
ScheduleHazardRecognizer::HazardType retval = NoHazard;
bool mustBeOdd = false;
Modified: llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.h (original)
+++ llvm/trunk/lib/Target/CellSPU/SPUHazardRecognizers.h Thu Dec 23 23:03:26 2010
@@ -30,7 +30,7 @@
public:
SPUHazardRecognizer(const TargetInstrInfo &TII);
- virtual HazardType getHazardType(SUnit *SU);
+ virtual HazardType getHazardType(SUnit *SU, int Stalls);
virtual void EmitInstruction(SUnit *SU);
virtual void AdvanceCycle();
virtual void EmitNoop();
Modified: llvm/trunk/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CellSPU/SPUISelDAGToDAG.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CellSPU/SPUISelDAGToDAG.cpp (original)
+++ llvm/trunk/lib/Target/CellSPU/SPUISelDAGToDAG.cpp Thu Dec 23 23:03:26 2010
@@ -301,14 +301,6 @@
return "Cell SPU DAG->DAG Pattern Instruction Selection";
}
- /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
- /// this target when scheduling the DAG.
- virtual ScheduleHazardRecognizer *CreateTargetHazardRecognizer() {
- const TargetInstrInfo *II = TM.getInstrInfo();
- assert(II && "No InstrInfo?");
- return new SPUHazardRecognizer(*II);
- }
-
private:
SDValue getRC( MVT );
Modified: llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.cpp (original)
+++ llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.cpp Thu Dec 23 23:03:26 2010
@@ -16,6 +16,7 @@
#include "SPUInstrBuilder.h"
#include "SPUTargetMachine.h"
#include "SPUGenInstrInfo.inc"
+#include "SPUHazardRecognizers.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
@@ -54,6 +55,16 @@
RI(*TM.getSubtargetImpl(), *this)
{ /* NOP */ }
+/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
+/// this target when scheduling the DAG.
+ScheduleHazardRecognizer *SPUInstrInfo::CreateTargetHazardRecognizer(
+ const TargetMachine *TM,
+ const ScheduleDAG *DAG) const {
+ const TargetInstrInfo *TII = TM->getInstrInfo();
+ assert(TII && "No InstrInfo?");
+ return new SPUHazardRecognizer(*TII);
+}
+
unsigned
SPUInstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
int &FrameIndex) const {
Modified: llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.h (original)
+++ llvm/trunk/lib/Target/CellSPU/SPUInstrInfo.h Thu Dec 23 23:03:26 2010
@@ -32,6 +32,10 @@
///
virtual const SPURegisterInfo &getRegisterInfo() const { return RI; }
+ ScheduleHazardRecognizer *
+ CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const;
+
unsigned isLoadFromStackSlot(const MachineInstr *MI,
int &FrameIndex) const;
unsigned isStoreToStackSlot(const MachineInstr *MI,
Modified: llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.cpp Thu Dec 23 23:03:26 2010
@@ -122,7 +122,9 @@
/// instructions that wouldn't terminate the dispatch group that would cause a
/// pipeline flush.
ScheduleHazardRecognizer::HazardType PPCHazardRecognizer970::
-getHazardType(SUnit *SU) {
+getHazardType(SUnit *SU, int Stalls) {
+ assert(Stalls == 0 && "PPC hazards don't support scoreboard lookahead");
+
const SDNode *Node = SU->getNode()->getGluedMachineNode();
bool isFirst, isSingle, isCracked, isLoad, isStore;
PPCII::PPC970_Unit InstrType =
Modified: llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.h (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCHazardRecognizers.h Thu Dec 23 23:03:26 2010
@@ -48,7 +48,7 @@
public:
PPCHazardRecognizer970(const TargetInstrInfo &TII);
- virtual HazardType getHazardType(SUnit *SU);
+ virtual HazardType getHazardType(SUnit *SU, int Stalls);
virtual void EmitInstruction(SUnit *SU);
virtual void AdvanceCycle();
Modified: llvm/trunk/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCISelDAGToDAG.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCISelDAGToDAG.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCISelDAGToDAG.cpp Thu Dec 23 23:03:26 2010
@@ -16,7 +16,6 @@
#include "PPC.h"
#include "PPCPredicates.h"
#include "PPCTargetMachine.h"
-#include "PPCHazardRecognizers.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionAnalysis.h"
@@ -155,16 +154,6 @@
return "PowerPC DAG->DAG Pattern Instruction Selection";
}
- /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
- /// this target when scheduling the DAG.
- virtual ScheduleHazardRecognizer *CreateTargetHazardRecognizer() {
- // Should use subtarget info to pick the right hazard recognizer. For
- // now, always return a PPC970 recognizer.
- const TargetInstrInfo *II = TM.getInstrInfo();
- assert(II && "No InstrInfo?");
- return new PPCHazardRecognizer970(*II);
- }
-
// Include the pieces autogenerated from the target description.
#include "PPCGenDAGISel.inc"
Modified: llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.cpp?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.cpp Thu Dec 23 23:03:26 2010
@@ -17,6 +17,7 @@
#include "PPCPredicates.h"
#include "PPCGenInstrInfo.inc"
#include "PPCTargetMachine.h"
+#include "PPCHazardRecognizers.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -39,6 +40,18 @@
: TargetInstrInfoImpl(PPCInsts, array_lengthof(PPCInsts)), TM(tm),
RI(*TM.getSubtargetImpl(), *this) {}
+/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
+/// this target when scheduling the DAG.
+ScheduleHazardRecognizer *PPCInstrInfo::CreateTargetHazardRecognizer(
+ const TargetMachine *TM,
+ const ScheduleDAG *DAG) const {
+ // Should use subtarget info to pick the right hazard recognizer. For
+ // now, always return a PPC970 recognizer.
+ const TargetInstrInfo *TII = TM->getInstrInfo();
+ assert(TII && "No InstrInfo?");
+ return new PPCHazardRecognizer970(*TII);
+}
+
unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
int &FrameIndex) const {
switch (MI->getOpcode()) {
Modified: llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.h?rev=122541&r1=122540&r2=122541&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.h (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCInstrInfo.h Thu Dec 23 23:03:26 2010
@@ -82,6 +82,10 @@
///
virtual const PPCRegisterInfo &getRegisterInfo() const { return RI; }
+ ScheduleHazardRecognizer *
+ CreateTargetHazardRecognizer(const TargetMachine *TM,
+ const ScheduleDAG *DAG) const;
+
unsigned isLoadFromStackSlot(const MachineInstr *MI,
int &FrameIndex) const;
unsigned isStoreToStackSlot(const MachineInstr *MI,
More information about the llvm-commits
mailing list