[llvm] [MachinePipeliner] Add validation for missed dependencies (PR #135148)

[Ryotaro Kasuga via llvm-commits]

================
@@ -3922,41 +4187,202 @@ SwingSchedulerDDG::getEdges(const SUnit *SU) const {
 void SwingSchedulerDDG::addEdge(const SUnit *SU,
                                 const SwingSchedulerDDGEdge &Edge) {
   auto &Edges = getEdges(SU);
-  if (Edge.getSrc() == SU)
-    Edges.Succs.push_back(Edge);
-  else
-    Edges.Preds.push_back(Edge);
+  if (Edge.isValidationOnly()) {
+    assert(SU == Edge.getDst() && "Validation only edges must be added to the "
+                                  "destination node");
+    Edges.ValidationOnlyPreds.push_back(Edge);
+  } else {
+    if (Edge.getSrc() == SU)
+      Edges.Succs.append(Edge);
+    else
+      Edges.Preds.append(Edge);
+  }
 }
 
 void SwingSchedulerDDG::initEdges(SUnit *SU) {
   for (const auto &PI : SU->Preds) {
-    SwingSchedulerDDGEdge Edge(SU, PI, false);
+    SwingSchedulerDDGEdge Edge(SU, PI, false, false);
     addEdge(SU, Edge);
   }
 
   for (const auto &SI : SU->Succs) {
-    SwingSchedulerDDGEdge Edge(SU, SI, true);
+    SwingSchedulerDDGEdge Edge(SU, SI, true, false);
     addEdge(SU, Edge);
   }
 }
 
 SwingSchedulerDDG::SwingSchedulerDDG(std::vector<SUnit> &SUnits, SUnit *EntrySU,
-                                     SUnit *ExitSU)
+                                     SUnit *ExitSU, const LoopCarriedEdges &LCE)
     : EntrySU(EntrySU), ExitSU(ExitSU) {
   EdgesVec.resize(SUnits.size());
 
   initEdges(EntrySU);
   initEdges(ExitSU);
   for (auto &SU : SUnits)
     initEdges(&SU);
+
+  for (SUnit &SU : SUnits) {
+    SUnit *Src = &SU;
+
+    if (auto *OutputDep = LCE.getOutputDepOrNull(Src))
+      for (const auto &[Reg, Set] : *OutputDep) {
+        SDep Dep(Src, SDep::Output, Reg);
+        Dep.setLatency(1);
+        for (SUnit *Dst : Set) {
+          SwingSchedulerDDGEdge Edge(Dst, Dep, /*IsSucc=*/false,
+                                     /*IsValidationOnly=*/true);
+          Edge.setDistance(1);
+          addEdge(Dst, Edge);
+        }
+      }
+
+    if (auto *OrderDep = LCE.getOrderDepOrNull(Src)) {
+      SDep Dep(Src, SDep::Barrier);
+      Dep.setLatency(1);
+      for (SUnit *Dst : *OrderDep) {
+        SwingSchedulerDDGEdge Edge(
+            Dst, Dep, /*IsSucc=*/false,
+            /*IsValidationOnly=*/!LCE.shouldUseWhenScheduling(Src, Dst));
+        Edge.setDistance(1);
+
+        // If this edge is used when scheduling, this should be added both Preds
+        // and Succs.
+        if (!Edge.isValidationOnly())
+          addEdge(Src, Edge);
+        addEdge(Dst, Edge);
+      }
+    }
+  }
 }
 
-const SwingSchedulerDDG::EdgesType &
+ArrayRef<SwingSchedulerDDGEdge>
 SwingSchedulerDDG::getInEdges(const SUnit *SU) const {
-  return getEdges(SU).Preds;
+  return getEdges(SU).Preds.get(UseLoopCarriedEdges);
 }
 
-const SwingSchedulerDDG::EdgesType &
+ArrayRef<SwingSchedulerDDGEdge>
 SwingSchedulerDDG::getOutEdges(const SUnit *SU) const {
-  return getEdges(SU).Succs;
+  return getEdges(SU).Succs.get(UseLoopCarriedEdges);
+}
+
+bool SwingSchedulerDDG::isValidSchedule(std::vector<SUnit> &SUnits,
+                                        const SMSchedule &Schedule) const {
+  unsigned II = Schedule.getInitiationInterval();
+
+  auto ExpandedCycle = [&](SUnit *SU) {
+    int Stage = Schedule.stageScheduled(SU);
+    int Cycle = Schedule.cycleScheduled(SU);
+    return Cycle + (Stage * II);
+  };
+
+  for (SUnit &Dst : SUnits) {
+    if (!Dst.isInstr())
+      continue;
+    int CycleDst = ExpandedCycle(&Dst);
+    for (const SwingSchedulerDDGEdge &Edge :
+         getEdges(&Dst).ValidationOnlyPreds) {
+      SUnit *Src = Edge.getSrc();
+      if (!Src->isInstr())
+        continue;
+      int CycleSrc = ExpandedCycle(Src);
+      int MaxLateStart = CycleDst + Edge.getDistance() * II - Edge.getLatency();
+      if (CycleSrc > MaxLateStart) {
+        LLVM_DEBUG({
+          dbgs() << "Validation failed for edge from " << Src->NodeNum << " to "
+                 << Dst.NodeNum << "\n";
+        });
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+void SwingSchedulerDDG::EdgesType::append(const SwingSchedulerDDGEdge &Edge) {
+  bool LoopCarriedOrderDep = Edge.isOrderDep() && Edge.getDistance() != 0;
+  assert(!(LoopCarriedOrderDepsCount != 0 && !LoopCarriedOrderDep) &&
+         "Loop-carried edges should not be added to the underlying edges");
+  Underlying.push_back(Edge);
+  if (LoopCarriedOrderDep)
+    ++LoopCarriedOrderDepsCount;
+}
+
+bool LoopCarriedEdges::shouldUseWhenScheduling(const SUnit *From,
+                                               const SUnit *To) const {
+  if (From->NodeNum < To->NodeNum)
+    return false;
+  auto Ite = BackEdges.find(From);
+  if (Ite == BackEdges.end())
+    return false;
+  return Ite->second.contains(To);
+}
+
+void LoopCarriedEdges::modifySUnits(std::vector<SUnit> &SUnits,
+                                    const TargetInstrInfo *TII) {
+  for (SUnit &SU : SUnits) {
+    SUnit *Src = &SU;
+    if (auto *OrderDep = getOrderDepOrNull(Src)) {
+      SDep Dep(Src, SDep::Barrier);
+      Dep.setLatency(1);
+      for (SUnit *Dst : *OrderDep) {
+        SUnit *From = Src;
+        SUnit *To = Dst;
+        bool IsBackEdge = Src->NodeNum > Dst->NodeNum;
+        if (IsBackEdge) {
+          BackEdges[From].insert(To);
+          std::swap(From, To);
+        }
+
+        // To keep the previouse behavior, add a foward direction edge if the
+        // following conditions are met:
+        //
+        // - From is a load and To is a store.
+        // - The load apprears before the store in the original basic block.
+        // - There is no barrier/store between the two instructions.
+        // - We cannot reach to the store from the load through current edges in
+        //   the DAG.
----------------
kasuga-fj wrote:

These are what the previous implementation added in `addLoopCarriedDependencies`. Essentially, other types of dependencies (store-to-store, store-to-load) need to be added as well.

https://github.com/llvm/llvm-project/pull/135148