[llvm-branch-commits] [llvm-branch] r85940 - in /llvm/branches/Apple/Leela: include/llvm/CodeGen/LatencyPriorityQueue.h include/llvm/CodeGen/ScheduleDAG.h lib/CodeGen/AggressiveAntiDepBreaker.cpp lib/CodeGen/AggressiveAntiDepBreaker.h lib/CodeGen/AntiDepBreaker.h lib/CodeGen/CriticalAntiDepBreaker.cpp lib/CodeGen/CriticalAntiDepBreaker.h lib/CodeGen/ExactHazardRecognizer.cpp lib/CodeGen/LatencyPriorityQueue.cpp lib/CodeGen/PostRASchedulerList.cpp lib/CodeGen/ScheduleDAG.cpp lib/CodeGen/ScheduleDAGInstrs.cpp

Bill Wendling isanbard at gmail.com
Tue Nov 3 13:03:02 PST 2009


Author: void
Date: Tue Nov  3 15:03:02 2009
New Revision: 85940

URL: http://llvm.org/viewvc/llvm-project?rev=85940&view=rev
Log:
$ svn merge -c 85934 https://llvm.org/svn/llvm-project/llvm/trunk
--- Merging r85934 into '.':
U    lib/CodeGen/ScheduleDAGInstrs.cpp

$ svn merge -c 85939 https://llvm.org/svn/llvm-project/llvm/trunk
--- Merging r85939 into '.':
U    include/llvm/CodeGen/LatencyPriorityQueue.h
U    include/llvm/CodeGen/ScheduleDAG.h
U    lib/CodeGen/CriticalAntiDepBreaker.h
U    lib/CodeGen/ScheduleDAG.cpp
U    lib/CodeGen/CriticalAntiDepBreaker.cpp
U    lib/CodeGen/LatencyPriorityQueue.cpp
U    lib/CodeGen/ExactHazardRecognizer.cpp
U    lib/CodeGen/AggressiveAntiDepBreaker.cpp
U    lib/CodeGen/PostRASchedulerList.cpp
U    lib/CodeGen/AntiDepBreaker.h
U    lib/CodeGen/AggressiveAntiDepBreaker.h


Modified:
    llvm/branches/Apple/Leela/include/llvm/CodeGen/LatencyPriorityQueue.h
    llvm/branches/Apple/Leela/include/llvm/CodeGen/ScheduleDAG.h
    llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.h
    llvm/branches/Apple/Leela/lib/CodeGen/AntiDepBreaker.h
    llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.h
    llvm/branches/Apple/Leela/lib/CodeGen/ExactHazardRecognizer.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/LatencyPriorityQueue.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/PostRASchedulerList.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAG.cpp
    llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAGInstrs.cpp

Modified: llvm/branches/Apple/Leela/include/llvm/CodeGen/LatencyPriorityQueue.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/include/llvm/CodeGen/LatencyPriorityQueue.h?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/include/llvm/CodeGen/LatencyPriorityQueue.h (original)
+++ llvm/branches/Apple/Leela/include/llvm/CodeGen/LatencyPriorityQueue.h Tue Nov  3 15:03:02 2009
@@ -39,12 +39,21 @@
     /// predecessor for.  This is used as a tie-breaker heuristic for better
     /// mobility.
     std::vector<unsigned> NumNodesSolelyBlocking;
-
+    
+    /// IgnoreAntiDep - Ignore anti-dependencies
+    bool IgnoreAntiDep;
+    
+    /// Queue - The queue.
     PriorityQueue<SUnit*, std::vector<SUnit*>, latency_sort> Queue;
+
 public:
-    LatencyPriorityQueue() : Queue(latency_sort(this)) {
+  LatencyPriorityQueue() : IgnoreAntiDep(false), Queue(latency_sort(this)) {
     }
-    
+
+    void setIgnoreAntiDep(bool ignore) {
+      IgnoreAntiDep = ignore;
+    }
+
     void initNodes(std::vector<SUnit> &sunits) {
       SUnits = &sunits;
       NumNodesSolelyBlocking.resize(SUnits->size(), 0);
@@ -63,7 +72,7 @@
     
     unsigned getLatency(unsigned NodeNum) const {
       assert(NodeNum < (*SUnits).size());
-      return (*SUnits)[NodeNum].getHeight();
+      return (*SUnits)[NodeNum].getHeight(IgnoreAntiDep);
     }
     
     unsigned getNumSolelyBlockNodes(unsigned NodeNum) const {

Modified: llvm/branches/Apple/Leela/include/llvm/CodeGen/ScheduleDAG.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/include/llvm/CodeGen/ScheduleDAG.h?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/include/llvm/CodeGen/ScheduleDAG.h (original)
+++ llvm/branches/Apple/Leela/include/llvm/CodeGen/ScheduleDAG.h Tue Nov  3 15:03:02 2009
@@ -340,28 +340,34 @@
     void removePred(const SDep &D);
 
     /// getDepth - Return the depth of this node, which is the length of the
-    /// maximum path up to any node with has no predecessors.
-    unsigned getDepth() const {
-      if (!isDepthCurrent) const_cast<SUnit *>(this)->ComputeDepth();
+    /// maximum path up to any node with has no predecessors. If IgnoreAntiDep
+    /// is true, ignore anti-dependence edges.
+    unsigned getDepth(bool IgnoreAntiDep=false) const {
+      if (!isDepthCurrent) 
+        const_cast<SUnit *>(this)->ComputeDepth(IgnoreAntiDep);
       return Depth;
     }
 
     /// getHeight - Return the height of this node, which is the length of the
-    /// maximum path down to any node with has no successors.
-    unsigned getHeight() const {
-      if (!isHeightCurrent) const_cast<SUnit *>(this)->ComputeHeight();
+    /// maximum path down to any node with has no successors. If IgnoreAntiDep
+    /// is true, ignore anti-dependence edges.
+    unsigned getHeight(bool IgnoreAntiDep=false) const {
+      if (!isHeightCurrent) 
+        const_cast<SUnit *>(this)->ComputeHeight(IgnoreAntiDep);
       return Height;
     }
 
-    /// setDepthToAtLeast - If NewDepth is greater than this node's depth
-    /// value, set it to be the new depth value. This also recursively
-    /// marks successor nodes dirty.
-    void setDepthToAtLeast(unsigned NewDepth);
-
-    /// setDepthToAtLeast - If NewDepth is greater than this node's depth
-    /// value, set it to be the new height value. This also recursively
-    /// marks predecessor nodes dirty.
-    void setHeightToAtLeast(unsigned NewHeight);
+    /// setDepthToAtLeast - If NewDepth is greater than this node's
+    /// depth value, set it to be the new depth value. This also
+    /// recursively marks successor nodes dirty.  If IgnoreAntiDep is
+    /// true, ignore anti-dependence edges.
+    void setDepthToAtLeast(unsigned NewDepth, bool IgnoreAntiDep=false);
+
+    /// setDepthToAtLeast - If NewDepth is greater than this node's
+    /// depth value, set it to be the new height value. This also
+    /// recursively marks predecessor nodes dirty. If IgnoreAntiDep is
+    /// true, ignore anti-dependence edges.
+    void setHeightToAtLeast(unsigned NewHeight, bool IgnoreAntiDep=false);
 
     /// setDepthDirty - Set a flag in this node to indicate that its
     /// stored Depth value will require recomputation the next time
@@ -394,8 +400,8 @@
     void print(raw_ostream &O, const ScheduleDAG *G) const;
 
   private:
-    void ComputeDepth();
-    void ComputeHeight();
+    void ComputeDepth(bool IgnoreAntiDep);
+    void ComputeHeight(bool IgnoreAntiDep);
   };
 
   //===--------------------------------------------------------------------===//

Modified: llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.cpp Tue Nov  3 15:03:02 2009
@@ -14,7 +14,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "aggressive-antidep"
+#define DEBUG_TYPE "post-RA-sched"
 #include "AggressiveAntiDepBreaker.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -31,7 +31,7 @@
 static cl::opt<int>
 AntiDepTrials("agg-antidep-trials",
               cl::desc("Maximum number of anti-dependency breaking passes"),
-              cl::init(2), cl::Hidden);
+              cl::init(1), cl::Hidden);
 
 AggressiveAntiDepState::AggressiveAntiDepState(MachineBasicBlock *BB) :
   GroupNodes(TargetRegisterInfo::FirstVirtualRegister, 0) {
@@ -265,18 +265,24 @@
 }
 
 /// AntiDepPathStep - Return SUnit that SU has an anti-dependence on.
-static void AntiDepPathStep(SUnit *SU, std::vector<SDep*>& Edges) {
-  SmallSet<unsigned, 8> Dups;
+static void AntiDepPathStep(SUnit *SU, AntiDepBreaker::AntiDepRegVector& Regs,
+                            std::vector<SDep*>& Edges) {
+  AntiDepBreaker::AntiDepRegSet RegSet;
+  for (unsigned i = 0, e = Regs.size(); i < e; ++i)
+    RegSet.insert(Regs[i]);
+
   for (SUnit::pred_iterator P = SU->Preds.begin(), PE = SU->Preds.end();
        P != PE; ++P) {
     if (P->getKind() == SDep::Anti) {
       unsigned Reg = P->getReg();
-      if (Dups.count(Reg) == 0) {
+      if (RegSet.count(Reg) != 0) {
         Edges.push_back(&*P);
-        Dups.insert(Reg);
+        RegSet.erase(Reg);
       }
     }
   }
+
+  assert(RegSet.empty() && "Expected all antidep registers to be found");
 }
 
 void AggressiveAntiDepBreaker::HandleLastUse(unsigned Reg, unsigned KillIdx,
@@ -593,6 +599,7 @@
 ///
 unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
                               std::vector<SUnit>& SUnits,
+                              CandidateMap& Candidates,
                               MachineBasicBlock::iterator& Begin,
                               MachineBasicBlock::iterator& End,
                               unsigned InsertPosIndex) {
@@ -601,9 +608,15 @@
   std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>& 
     RegRefs = State->GetRegRefs();
 
+  // Nothing to do if no candidates.
+  if (Candidates.empty()) {
+    DEBUG(errs() << "\n===== No anti-dependency candidates\n");
+    return 0;
+  }
+
   // The code below assumes that there is at least one instruction,
   // so just duck out immediately if the block is empty.
-  if (SUnits.empty()) return false;
+  if (SUnits.empty()) return 0;
   
   // Manage saved state to enable multiple passes...
   if (AntiDepTrials > 1) {
@@ -618,7 +631,8 @@
   // ...need a map from MI to SUnit.
   std::map<MachineInstr *, SUnit *> MISUnitMap;
 
-  DEBUG(errs() << "Breaking all anti-dependencies\n");
+  DEBUG(errs() << "\n===== Attempting to break " << Candidates.size() << 
+        " anti-dependencies\n");
   for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
     SUnit *SU = &SUnits[i];
     MISUnitMap.insert(std::pair<MachineInstr *, SUnit *>(SU->getInstr(), SU));
@@ -655,8 +669,10 @@
 
     std::vector<SDep*> Edges;
     SUnit *PathSU = MISUnitMap[MI];
-    if (PathSU) 
-      AntiDepPathStep(PathSU, Edges);
+    AntiDepBreaker::CandidateMap::iterator 
+      citer = Candidates.find(PathSU);
+    if (citer != Candidates.end())
+      AntiDepPathStep(PathSU, citer->second, Edges);
       
     // Ignore KILL instructions (they form a group in ScanInstruction
     // but don't cause any anti-dependence breaking themselves)

Modified: llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.h?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.h (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/AggressiveAntiDepBreaker.h Tue Nov  3 15:03:02 2009
@@ -131,6 +131,9 @@
     /// dependencies may be exposed, so multiple passes are required.
     unsigned GetMaxTrials();
 
+    /// NeedCandidates - Candidates required.
+    bool NeedCandidates() { return true; }
+
     /// Start - Initialize anti-dep breaking for a new basic block.
     void StartBlock(MachineBasicBlock *BB);
 
@@ -138,6 +141,7 @@
     /// of the ScheduleDAG and break them by renaming registers.
     ///
     unsigned BreakAntiDependencies(std::vector<SUnit>& SUnits,
+                                   CandidateMap& Candidates,
                                    MachineBasicBlock::iterator& Begin,
                                    MachineBasicBlock::iterator& End,
                                    unsigned InsertPosIndex);

Modified: llvm/branches/Apple/Leela/lib/CodeGen/AntiDepBreaker.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/AntiDepBreaker.h?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/AntiDepBreaker.h (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/AntiDepBreaker.h Tue Nov  3 15:03:02 2009
@@ -21,6 +21,8 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
 
@@ -29,12 +31,20 @@
 /// anti-dependencies.
 class AntiDepBreaker {
 public:
+  typedef SmallSet<unsigned, 4> AntiDepRegSet;
+  typedef SmallVector<unsigned, 4> AntiDepRegVector;
+  typedef std::map<SUnit *, AntiDepRegVector> CandidateMap;
+
   virtual ~AntiDepBreaker();
 
   /// GetMaxTrials - Return the maximum number of anti-dependence
   /// breaking attempts that will be made for a block.
   virtual unsigned GetMaxTrials() =0;
 
+  /// NeedCandidates - Return true if the schedule must provide
+  /// candidates with BreakAntiDependencies().
+  virtual bool NeedCandidates() =0;
+
   /// Start - Initialize anti-dep breaking for a new basic block.
   virtual void StartBlock(MachineBasicBlock *BB) =0;
 
@@ -43,9 +53,10 @@
   /// the number of anti-dependencies broken.
   ///
   virtual unsigned BreakAntiDependencies(std::vector<SUnit>& SUnits,
-                                         MachineBasicBlock::iterator& Begin,
-                                         MachineBasicBlock::iterator& End,
-                                         unsigned InsertPosIndex) =0;
+                                CandidateMap& Candidates,
+                                MachineBasicBlock::iterator& Begin,
+                                MachineBasicBlock::iterator& End,
+                                unsigned InsertPosIndex) =0;
   
   /// Observe - Update liveness information to account for the current
   /// instruction, which will not be scheduled.

Modified: llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.cpp Tue Nov  3 15:03:02 2009
@@ -13,7 +13,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "critical-antidep"
+#define DEBUG_TYPE "post-RA-sched"
 #include "CriticalAntiDepBreaker.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -316,6 +316,7 @@
 
 unsigned CriticalAntiDepBreaker::
 BreakAntiDependencies(std::vector<SUnit>& SUnits,
+                      CandidateMap& Candidates,
                       MachineBasicBlock::iterator& Begin,
                       MachineBasicBlock::iterator& End,
                       unsigned InsertPosIndex) {

Modified: llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.h?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.h (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/CriticalAntiDepBreaker.h Tue Nov  3 15:03:02 2009
@@ -68,6 +68,9 @@
     /// only a single pass
     unsigned GetMaxTrials() { return 1; }
 
+    /// NeedCandidates - Candidates not needed.
+    bool NeedCandidates() { return false; }
+
     /// Start - Initialize anti-dep breaking for a new basic block.
     void StartBlock(MachineBasicBlock *BB);
 
@@ -75,6 +78,7 @@
     /// of the ScheduleDAG and break them by renaming registers.
     ///
     unsigned BreakAntiDependencies(std::vector<SUnit>& SUnits,
+                                   CandidateMap& Candidates,
                                    MachineBasicBlock::iterator& Begin,
                                    MachineBasicBlock::iterator& End,
                                    unsigned InsertPosIndex);

Modified: llvm/branches/Apple/Leela/lib/CodeGen/ExactHazardRecognizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/ExactHazardRecognizer.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/ExactHazardRecognizer.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/ExactHazardRecognizer.cpp Tue Nov  3 15:03:02 2009
@@ -12,7 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "exact-hazards"
+#define DEBUG_TYPE "post-RA-sched"
 #include "ExactHazardRecognizer.h"
 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
 #include "llvm/Support/Debug.h"

Modified: llvm/branches/Apple/Leela/lib/CodeGen/LatencyPriorityQueue.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/LatencyPriorityQueue.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/LatencyPriorityQueue.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/LatencyPriorityQueue.cpp Tue Nov  3 15:03:02 2009
@@ -55,6 +55,7 @@
   SUnit *OnlyAvailablePred = 0;
   for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
        I != E; ++I) {
+    if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
     SUnit &Pred = *I->getSUnit();
     if (!Pred.isScheduled) {
       // We found an available, but not scheduled, predecessor.  If it's the
@@ -73,9 +74,11 @@
   // this node is the sole unscheduled node for.
   unsigned NumNodesBlocking = 0;
   for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
-       I != E; ++I)
+       I != E; ++I) {
+    if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
     if (getSingleUnscheduledPred(I->getSUnit()) == SU)
       ++NumNodesBlocking;
+  }
   NumNodesSolelyBlocking[SU->NodeNum] = NumNodesBlocking;
   
   Queue.push(SU);
@@ -88,8 +91,10 @@
 // the node available.
 void LatencyPriorityQueue::ScheduledNode(SUnit *SU) {
   for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
-       I != E; ++I)
+       I != E; ++I) {
+    if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
     AdjustPriorityOfUnscheduledPreds(I->getSUnit());
+  }
 }
 
 /// AdjustPriorityOfUnscheduledPreds - One of the predecessors of SU was just

Modified: llvm/branches/Apple/Leela/lib/CodeGen/PostRASchedulerList.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/PostRASchedulerList.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/PostRASchedulerList.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/PostRASchedulerList.cpp Tue Nov  3 15:03:02 2009
@@ -175,10 +175,11 @@
     void FixupKills(MachineBasicBlock *MBB);
 
   private:
-    void ReleaseSucc(SUnit *SU, SDep *SuccEdge);
-    void ReleaseSuccessors(SUnit *SU);
-    void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
-    void ListScheduleTopDown();
+    void ReleaseSucc(SUnit *SU, SDep *SuccEdge, bool IgnoreAntiDep);
+    void ReleaseSuccessors(SUnit *SU, bool IgnoreAntiDep);
+    void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle, bool IgnoreAntiDep);
+    void ListScheduleTopDown(
+           AntiDepBreaker::CandidateMap *AntiDepCandidates);
     void StartBlockForKills(MachineBasicBlock *BB);
     
     // ToggleKillFlag - Toggle a register operand kill flag. Other
@@ -320,15 +321,32 @@
   BuildSchedGraph(AA);
 
   if (AntiDepBreak != NULL) {
+    AntiDepBreaker::CandidateMap AntiDepCandidates;
+    const bool NeedCandidates = AntiDepBreak->NeedCandidates();
+    
     for (unsigned i = 0, Trials = AntiDepBreak->GetMaxTrials();
          i < Trials; ++i) {
-      DEBUG(errs() << "********** Break Anti-Deps, Trial " << 
+      DEBUG(errs() << "\n********** Break Anti-Deps, Trial " << 
             i << " **********\n");
+      
+      // If candidates are required, then schedule forward ignoring
+      // anti-dependencies to collect the candidate operands for
+      // anti-dependence breaking. The candidates will be the def
+      // operands for the anti-dependencies that if broken would allow
+      // an improved schedule
+      if (NeedCandidates) {
+        DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+                SUnits[su].dumpAll(this));
+
+        AntiDepCandidates.clear();
+        AvailableQueue.initNodes(SUnits);
+        ListScheduleTopDown(&AntiDepCandidates);
+        AvailableQueue.releaseState();
+      }
+
       unsigned Broken = 
-        AntiDepBreak->BreakAntiDependencies(SUnits, Begin, InsertPos,
-                                            InsertPosIndex);
-      if (Broken == 0)
-        break;
+        AntiDepBreak->BreakAntiDependencies(SUnits, AntiDepCandidates,
+                                            Begin, InsertPos, InsertPosIndex);
 
       // We made changes. Update the dependency graph.
       // Theoretically we could update the graph in place:
@@ -336,24 +354,26 @@
       // the def's anti-dependence *and* output-dependence edges due to
       // that register, and add new anti-dependence and output-dependence
       // edges based on the next live range of the register.
-      SUnits.clear();
-      EntrySU = SUnit();
-      ExitSU = SUnit();
-      BuildSchedGraph(AA);
+      if ((Broken != 0) || NeedCandidates) {
+        SUnits.clear();
+        Sequence.clear();
+        EntrySU = SUnit();
+        ExitSU = SUnit();
+        BuildSchedGraph(AA);
+      }
 
       NumFixedAnti += Broken;
+      if (Broken == 0)
+        break;
     }
   }
 
   DEBUG(errs() << "********** List Scheduling **********\n");
-  
   DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
           SUnits[su].dumpAll(this));
 
   AvailableQueue.initNodes(SUnits);
-
-  ListScheduleTopDown();
-  
+  ListScheduleTopDown(NULL);
   AvailableQueue.releaseState();
 }
 
@@ -552,7 +572,8 @@
 
 /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to
 /// the PendingQueue if the count reaches zero. Also update its cycle bound.
-void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SDep *SuccEdge) {
+void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SDep *SuccEdge,
+                                       bool IgnoreAntiDep) {
   SUnit *SuccSU = SuccEdge->getSUnit();
 
 #ifndef NDEBUG
@@ -568,7 +589,8 @@
   // Compute how many cycles it will be before this actually becomes
   // available.  This is the max of the start time of all predecessors plus
   // their latencies.
-  SuccSU->setDepthToAtLeast(SU->getDepth() + SuccEdge->getLatency());
+  SuccSU->setDepthToAtLeast(SU->getDepth(IgnoreAntiDep) +
+                            SuccEdge->getLatency(), IgnoreAntiDep);
   
   // If all the node's predecessors are scheduled, this node is ready
   // to be scheduled. Ignore the special ExitSU node.
@@ -577,40 +599,73 @@
 }
 
 /// ReleaseSuccessors - Call ReleaseSucc on each of SU's successors.
-void SchedulePostRATDList::ReleaseSuccessors(SUnit *SU) {
+void SchedulePostRATDList::ReleaseSuccessors(SUnit *SU, bool IgnoreAntiDep) {
   for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
-       I != E; ++I)
-    ReleaseSucc(SU, &*I);
+       I != E; ++I) {
+    if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
+    ReleaseSucc(SU, &*I, IgnoreAntiDep);
+  }
 }
 
 /// ScheduleNodeTopDown - Add the node to the schedule. Decrement the pending
 /// count of its successors. If a successor pending count is zero, add it to
 /// the Available queue.
-void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) {
+void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle,
+                                               bool IgnoreAntiDep) {
   DEBUG(errs() << "*** Scheduling [" << CurCycle << "]: ");
   DEBUG(SU->dump(this));
   
   Sequence.push_back(SU);
-  assert(CurCycle >= SU->getDepth() && "Node scheduled above its depth!");
-  SU->setDepthToAtLeast(CurCycle);
+  assert(CurCycle >= SU->getDepth(IgnoreAntiDep) && 
+         "Node scheduled above its depth!");
+  SU->setDepthToAtLeast(CurCycle, IgnoreAntiDep);
 
-  ReleaseSuccessors(SU);
+  ReleaseSuccessors(SU, IgnoreAntiDep);
   SU->isScheduled = true;
   AvailableQueue.ScheduledNode(SU);
 }
 
 /// ListScheduleTopDown - The main loop of list scheduling for top-down
 /// schedulers.
-void SchedulePostRATDList::ListScheduleTopDown() {
+void SchedulePostRATDList::ListScheduleTopDown(
+                   AntiDepBreaker::CandidateMap *AntiDepCandidates) {
   unsigned CurCycle = 0;
+  const bool IgnoreAntiDep = (AntiDepCandidates != NULL);
+  
+  // We're scheduling top-down but we're visiting the regions in
+  // bottom-up order, so we don't know the hazards at the start of a
+  // region. So assume no hazards (this should usually be ok as most
+  // blocks are a single region).
+  HazardRec->Reset();
+
+  // If ignoring anti-dependencies, the Schedule DAG still has Anti
+  // dep edges, but we ignore them for scheduling purposes
+  AvailableQueue.setIgnoreAntiDep(IgnoreAntiDep);
 
   // Release any successors of the special Entry node.
-  ReleaseSuccessors(&EntrySU);
+  ReleaseSuccessors(&EntrySU, IgnoreAntiDep);
 
-  // All leaves to Available queue.
+  // Add all leaves to Available queue. If ignoring antideps we also
+  // adjust the predecessor count for each node to not include antidep
+  // edges.
   for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
     // It is available if it has no predecessors.
-    if (SUnits[i].Preds.empty()) {
+    bool available = SUnits[i].Preds.empty();
+    // If we are ignoring anti-dependencies then a node that has only
+    // anti-dep predecessors is available.
+    if (!available && IgnoreAntiDep) {
+      available = true;
+      for (SUnit::const_pred_iterator I = SUnits[i].Preds.begin(),
+             E = SUnits[i].Preds.end(); I != E; ++I) {
+        if (I->getKind() != SDep::Anti) {
+          available = false;
+        } else {
+          SUnits[i].NumPredsLeft -= 1;
+        }
+      }
+    }
+
+    if (available) {
       AvailableQueue.push(&SUnits[i]);
       SUnits[i].isAvailable = true;
     }
@@ -629,26 +684,25 @@
     // so, add them to the available queue.
     unsigned MinDepth = ~0u;
     for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) {
-      if (PendingQueue[i]->getDepth() <= CurCycle) {
+      if (PendingQueue[i]->getDepth(IgnoreAntiDep) <= CurCycle) {
         AvailableQueue.push(PendingQueue[i]);
         PendingQueue[i]->isAvailable = true;
         PendingQueue[i] = PendingQueue.back();
         PendingQueue.pop_back();
         --i; --e;
-      } else if (PendingQueue[i]->getDepth() < MinDepth)
-        MinDepth = PendingQueue[i]->getDepth();
+      } else if (PendingQueue[i]->getDepth(IgnoreAntiDep) < MinDepth)
+        MinDepth = PendingQueue[i]->getDepth(IgnoreAntiDep);
     }
 
     DEBUG(errs() << "\n*** Examining Available\n";
           LatencyPriorityQueue q = AvailableQueue;
           while (!q.empty()) {
             SUnit *su = q.pop();
-            errs() << "Height " << su->getHeight() << ": ";
+            errs() << "Height " << su->getHeight(IgnoreAntiDep) << ": ";
             su->dump(this);
           });
 
     SUnit *FoundSUnit = 0;
-
     bool HasNoopHazards = false;
     while (!AvailableQueue.empty()) {
       SUnit *CurSUnit = AvailableQueue.pop();
@@ -672,9 +726,30 @@
       NotReady.clear();
     }
 
-    // If we found a node to schedule, do it now.
+    // If we found a node to schedule...
     if (FoundSUnit) {
-      ScheduleNodeTopDown(FoundSUnit, CurCycle);
+      // If we are ignoring anti-dependencies and the SUnit we are
+      // scheduling has an antidep predecessor that has not been
+      // scheduled, then we will need to break that antidep if we want
+      // to get this schedule when not ignoring anti-dependencies.
+      if (IgnoreAntiDep) {
+        AntiDepBreaker::AntiDepRegVector AntiDepRegs;
+        for (SUnit::const_pred_iterator I = FoundSUnit->Preds.begin(),
+               E = FoundSUnit->Preds.end(); I != E; ++I) {
+          if ((I->getKind() == SDep::Anti) && !I->getSUnit()->isScheduled)
+            AntiDepRegs.push_back(I->getReg());
+        }
+        
+        if (AntiDepRegs.size() > 0) {
+          DEBUG(errs() << "*** AntiDep Candidate: ");
+          DEBUG(FoundSUnit->dump(this));
+          AntiDepCandidates->insert(
+            AntiDepBreaker::CandidateMap::value_type(FoundSUnit, AntiDepRegs));
+        }
+      }
+
+      // ... schedule the node...
+      ScheduleNodeTopDown(FoundSUnit, CurCycle, IgnoreAntiDep);
       HazardRec->EmitInstruction(FoundSUnit);
       CycleHasInsts = true;
 

Modified: llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAG.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAG.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAG.cpp Tue Nov  3 15:03:02 2009
@@ -183,8 +183,8 @@
 /// setDepthToAtLeast - Update this node's successors to reflect the
 /// fact that this node's depth just increased.
 ///
-void SUnit::setDepthToAtLeast(unsigned NewDepth) {
-  if (NewDepth <= getDepth())
+void SUnit::setDepthToAtLeast(unsigned NewDepth, bool IgnoreAntiDep) {
+  if (NewDepth <= getDepth(IgnoreAntiDep))
     return;
   setDepthDirty();
   Depth = NewDepth;
@@ -194,8 +194,8 @@
 /// setHeightToAtLeast - Update this node's predecessors to reflect the
 /// fact that this node's height just increased.
 ///
-void SUnit::setHeightToAtLeast(unsigned NewHeight) {
-  if (NewHeight <= getHeight())
+void SUnit::setHeightToAtLeast(unsigned NewHeight, bool IgnoreAntiDep) {
+  if (NewHeight <= getHeight(IgnoreAntiDep))
     return;
   setHeightDirty();
   Height = NewHeight;
@@ -204,7 +204,7 @@
 
 /// ComputeDepth - Calculate the maximal path from the node to the exit.
 ///
-void SUnit::ComputeDepth() {
+void SUnit::ComputeDepth(bool IgnoreAntiDep) {
   SmallVector<SUnit*, 8> WorkList;
   WorkList.push_back(this);
   do {
@@ -214,6 +214,7 @@
     unsigned MaxPredDepth = 0;
     for (SUnit::const_pred_iterator I = Cur->Preds.begin(),
          E = Cur->Preds.end(); I != E; ++I) {
+      if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
       SUnit *PredSU = I->getSUnit();
       if (PredSU->isDepthCurrent)
         MaxPredDepth = std::max(MaxPredDepth,
@@ -237,7 +238,7 @@
 
 /// ComputeHeight - Calculate the maximal path from the node to the entry.
 ///
-void SUnit::ComputeHeight() {
+void SUnit::ComputeHeight(bool IgnoreAntiDep) {
   SmallVector<SUnit*, 8> WorkList;
   WorkList.push_back(this);
   do {
@@ -247,6 +248,7 @@
     unsigned MaxSuccHeight = 0;
     for (SUnit::const_succ_iterator I = Cur->Succs.begin(),
          E = Cur->Succs.end(); I != E; ++I) {
+      if (IgnoreAntiDep && (I->getKind() == SDep::Anti)) continue;
       SUnit *SuccSU = I->getSUnit();
       if (SuccSU->isHeightCurrent)
         MaxSuccHeight = std::max(MaxSuccHeight,
@@ -346,7 +348,7 @@
       AnyNotSched = true;
     }
     if (SUnits[i].isScheduled &&
-        (isBottomUp ? SUnits[i].getHeight() : SUnits[i].getHeight()) >
+        (isBottomUp ? SUnits[i].getHeight() : SUnits[i].getDepth()) >
           unsigned(INT_MAX)) {
       if (!AnyNotSched)
         errs() << "*** Scheduling failed! ***\n";

Modified: llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAGInstrs.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAGInstrs.cpp?rev=85940&r1=85939&r2=85940&view=diff

==============================================================================
--- llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAGInstrs.cpp (original)
+++ llvm/branches/Apple/Leela/lib/CodeGen/ScheduleDAGInstrs.cpp Tue Nov  3 15:03:02 2009
@@ -98,7 +98,9 @@
 /// information and it can be tracked to a normal reference to a known
 /// object, return the Value for that object. Otherwise return null.
 static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI,
-                                                const MachineFrameInfo *MFI) {
+                                                const MachineFrameInfo *MFI,
+                                                bool &MayAlias) {
+  MayAlias = true;
   if (!MI->hasOneMemOperand() ||
       !(*MI->memoperands_begin())->getValue() ||
       (*MI->memoperands_begin())->isVolatile())
@@ -110,6 +112,7 @@
 
   V = getUnderlyingObject(V);
   if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) {
+    MayAlias = PSV->mayAlias(MFI);
     // For now, ignore PseudoSourceValues which may alias LLVM IR values
     // because the code that uses this function has no way to cope with
     // such aliases.
@@ -124,6 +127,23 @@
   return 0;
 }
 
+static bool mayUnderlyingObjectForInstrAlias(const MachineInstr *MI,
+                                             const MachineFrameInfo *MFI) {
+  if (!MI->hasOneMemOperand() ||
+      !(*MI->memoperands_begin())->getValue() ||
+      (*MI->memoperands_begin())->isVolatile())
+    return true;
+
+  const Value *V = (*MI->memoperands_begin())->getValue();
+  if (!V)
+    return true;
+
+  V = getUnderlyingObject(V);
+  if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V))
+    return PSV->mayAlias(MFI);
+  return true;
+}
+
 void ScheduleDAGInstrs::StartBlock(MachineBasicBlock *BB) {
   if (MachineLoop *ML = MLI.getLoopFor(BB))
     if (BB == ML->getLoopLatch()) {
@@ -362,8 +382,9 @@
         // Unknown memory accesses. Assume the worst.
         ChainMMO = 0;
     } else if (TID.mayStore()) {
+      bool MayAlias = true;
       TrueMemOrderLatency = STORE_LOAD_LATENCY;
-      if (const Value *V = getUnderlyingObjectForInstr(MI, MFI)) {
+      if (const Value *V = getUnderlyingObjectForInstr(MI, MFI, MayAlias)) {
         // A store to a specific PseudoSourceValue. Add precise dependencies.
         // Handle the def in MemDefs, if there is one.
         std::map<const Value *, SUnit *>::iterator I = MemDefs.find(V);
@@ -383,22 +404,26 @@
                                        /*Reg=*/0, /*isNormalMemory=*/true));
           J->second.clear();
         }
-        // Add dependencies from all the PendingLoads, since without
-        // memoperands we must assume they alias anything.
-        for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
-          PendingLoads[k]->addPred(SDep(SU, SDep::Order, TrueMemOrderLatency));
-        // Add a general dependence too, if needed.
-        if (Chain)
-          Chain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
-      } else {
+        if (MayAlias) {
+          // Add dependencies from all the PendingLoads, since without
+          // memoperands we must assume they alias anything.
+          for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
+            PendingLoads[k]->addPred(SDep(SU, SDep::Order, TrueMemOrderLatency));
+          // Add a general dependence too, if needed.
+          if (Chain)
+            Chain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+        }
+      } else if (MayAlias) {
         // Treat all other stores conservatively.
         goto new_chain;
       }
     } else if (TID.mayLoad()) {
+      bool MayAlias = true;
       TrueMemOrderLatency = 0;
       if (MI->isInvariantLoad(AA)) {
         // Invariant load, no chain dependencies needed!
-      } else if (const Value *V = getUnderlyingObjectForInstr(MI, MFI)) {
+      } else if (const Value *V = 
+                     getUnderlyingObjectForInstr(MI, MFI, MayAlias)) {
         // A load from a specific PseudoSourceValue. Add precise dependencies.
         std::map<const Value *, SUnit *>::iterator I = MemDefs.find(V);
         if (I != MemDefs.end())
@@ -414,16 +439,19 @@
         // Treat volatile loads conservatively. Note that this includes
         // cases where memoperand information is unavailable.
         goto new_chain;
-      } else {
-        // A normal load. Depend on the general chain, as well as on
+      } else if (MayAlias) {
+        // A "MayAlias" load. Depend on the general chain, as well as on
         // all stores. In the absense of MachineMemOperand information,
         // we can't even assume that the load doesn't alias well-behaved
         // memory locations.
         if (Chain)
           Chain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
         for (std::map<const Value *, SUnit *>::iterator I = MemDefs.begin(),
-             E = MemDefs.end(); I != E; ++I)
-          I->second->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+               E = MemDefs.end(); I != E; ++I) {
+          SUnit *DefSU = I->second;
+          if (mayUnderlyingObjectForInstrAlias(DefSU->getInstr(), MFI))
+            DefSU->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+        }
         PendingLoads.push_back(SU);
       }
     }





More information about the llvm-branch-commits mailing list