[llvm] [AMDGPU][NewInsertWaitcnt] Add Event Tracker (PR #226970)

[Matt Arsenault via llvm-commits]

================
@@ -0,0 +1,477 @@
+//===- AMDGPUEventTracking.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUEventTracking.h"
+#include "AMDGPUHWEvents.h"
+#include "AMDGPUWaitcntUtils.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include <algorithm>
+#include <optional>
+
+#define DEBUG_TYPE "amdgpu-event-tracking"
+
+namespace llvm {
+
+/// Mimic legacy (coarse) tracking of counter state.
+static cl::opt<bool> MimicLegacyTracking("amdgpu-event-legacy-tracking",
+                                         cl::init(false));
+
+#ifndef NDEBUG
+static cl::opt<bool> EventTrackerPrintAll(
+    "amdgpu-event-tracker-print-all", cl::init(false),
+    cl::desc("When using -debug, print the full set of live events every time "
+             "an event is added or removed"));
+#endif
+
+namespace AMDGPU {
+namespace eventtracking {
+
+namespace {
+static bool greaterThan(const EventTrackerRecord &A,
+                        const EventTrackerRecord &B) {
+  return A.getScore() > B.getScore();
+}
+} // namespace
+
+EventTrackerRecord::EventTrackerRecord(EventTrackingContext &Ctx,
+                                       MachineInstr *MI, SingleHWEvent Kind,
+                                       uint32_t Score)
+    : EventTrackerRecord(Ctx.nextDynamicInstanceID(), MI, Kind, Score) {}
+
+void EventTrackerRecord::print(raw_ostream &OS, bool PrintMI,
+                               unsigned Indent) const {
+  OS.indent(Indent) << "#" << ID << " " << Kind << " (Score=" << Score << "): ";
+  if (PrintMI && MI)
+    OS << *MI;
+  else
+    OS << "MI@" << (void *)MI << "\n";
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void EventTrackerRecord::dump() const {
+  dbgs() << "\n";
+  print(dbgs(), /*PrintMI=*/true);
+  dbgs() << "\n";
+}
+#endif
+
+EventTracker::EventTracker(MachineBasicBlock &MBB, EventTrackingContext &ETC)
+    : MBB(&MBB), Ctx(&ETC) {
+  Counters.resize(ETC.counters().size());
+  for (const CounterInfo &Info : ETC.counters())
+    Counters[Info.CounterT].CI = &Info;
+}
+
+void EventTracker::enterBlock() {
+  LLVM_DEBUG(dbgs() << "\n[EventTracker] Entering ";
+             MBB->printAsOperand(dbgs()); dbgs() << "\n");
+
+  // FIXME: This is a bit hacky, but we need to save the old state in case the
+  // MBB is also its own predecessor. Revisit when the design and clients of
+  // this class are set in stone.
+
+  SmallVector<EventTracker *> Preds;
+  bool IsSelfPred = false;
+  if (Preds.empty() && !MBB->pred_empty()) {
+    for (MachineBasicBlock *Pred : MBB->predecessors()) {
+      if (Pred == MBB)
+        IsSelfPred = true;
+      else
+        Preds.push_back(&(*Ctx)[Pred]);
+    }
+  }
+
+  if (IsSelfPred) {
+    EventTracker SelfCopy = *this;
+    Preds.push_back(&SelfCopy);
+    clear();
+    recordIncomings(*Ctx, Preds);
+  } else {
+    clear();
+    recordIncomings(*Ctx, Preds);
+  }
+}
+
+void EventTracker::leaveBlock() {
+  LLVM_DEBUG(dbgs() << "[EventTracker] Leaving "; MBB->printAsOperand(dbgs());
+             dbgs() << "\n");
+}
+
+void EventTracker::record(MachineInstr &MI, SingleHWEvent Event) {
+  LLVM_DEBUG(dbgs() << "[EventTracker] Recording " << Event << ": " << MI);
+
+  EventTrackerRecord Rec = EventTrackerRecord(*Ctx, &MI, Event);
+  [[maybe_unused]] bool FoundMatch = false;
+  for (auto &CD : Counters) {
+    if (!CD.CI->Events.contains(Event))
+      continue;
+
+    FoundMatch = true;
+    ++CD.Count;
+    CD.LegacyPendingEvents |= Event;
+
+    // Do not age records if we are out-of-order.
+    if (!CD.IsOutOfOrder) {
+      // NB: There is an intentional tradeoff here. We could avoid this loop by
+      // instead storing a timestamp in each record, and having a
+      // constantly-increasing clock to infer the score (clock-timestamp is
+      // score). However, it'd:
+      //  - Complexify fetching the score (`EventTrackerRecord` cannot answer it
+      //    on its own anymore and we need a separate query/wrapper).
+      //  - Make merge of incoming records a bit more annoying (we'd need to
+      //    rebase the `clock`).
+      //  - Potentially demand (much) more space in EventTrackerRecord to store
+      //    bigger numbers.
+      //
+      // All in all, I think this small loop is fine for now, but we can still
+      // change the system if we have data backed up by profiling to
+      // justify the change.
+      for (auto &Live : CD.LiveRecords)
+        Live.setScore(Live.getScore() + 1); // Age all existing events.
+    }
+
+    CD.LiveRecords.push_back(Rec);
+
+#ifndef NDEBUG
+    LLVM_DEBUG(if (EventTrackerPrintAll) {
+      dbgs().indent(2) << "Updated Timeline:\n";
+      print(dbgs(), CD, /*Indent=*/4);
+    });
+#endif
+  }
+
+  assert(FoundMatch && "Event has no matching InstCounterType!");
+}
+
+void EventTracker::wait(InstCounterType T, unsigned N) {
+  auto &CD = get(T);
+  LLVM_DEBUG(dbgs() << "[EventTracker] Wait on " << getInstCounterName(T)
+                    << " for " << N << "\n");
+
+  // Fast path for clearing the counter
+  if (N == 0) {
+    CD.LiveRecords.clear();
+    CD.Count = 0;
+    CD.IsIndeterminate = false;
+    CD.IsOutOfOrder = false;
+    CD.LegacyPendingEvents = HWEvents();
+    return;
+  }
+
+  CD.Count = std::min(CD.Count, N);
+
+  // Don't bother erasing stuff if we are out-of-order. All records have a score
+  // of zero in such cases.
+  if (!CD.IsOutOfOrder) {
+    auto *RmIt = remove_if(CD.LiveRecords, [&](EventTrackerRecord &E) {
+      if (E.getScore() < N)
+        return false;
+      LLVM_DEBUG(dbgs() << "  | Removing "; E.print(dbgs()));
+      return true;
+    });
+    CD.LiveRecords.erase(RmIt, CD.LiveRecords.end());
+  }
+
+  LLVM_DEBUG(dbgs() << "  | => Updated Count:" << CD.Count << "\n");
+
+#ifndef NDEBUG
+  LLVM_DEBUG(if (EventTrackerPrintAll) {
+    dbgs().indent(2) << "Updated Timeline:\n";
+    print(dbgs(), CD, /*Indent=*/4);
+  });
+#endif
+}
+
+void EventTracker::markIndeterminate(InstCounterType T) {
+  LLVM_DEBUG(dbgs() << "[EventTracker] Marking " << getInstCounterName(T)
+                    << " as indeterminate!\n");
+  auto &CD = get(T);
+  CD.IsIndeterminate = true;
+  markOutOfOrder(T);
+}
+
+void EventTracker::markOutOfOrder(InstCounterType T) {
+  LLVM_DEBUG(dbgs() << "[EventTracker] Marking " << getInstCounterName(T)
+                    << " as out-of-order!\n");
+  auto &CD = get(T);
+  CD.IsOutOfOrder = true;
+  for (auto &Rec : CD.LiveRecords)
+    Rec.setScore(0);
+}
+
+std::optional<unsigned> EventTracker::count(InstCounterType T) const {
+  auto &CD = get(T);
----------------
arsenm wrote:

no auto 

https://github.com/llvm/llvm-project/pull/226970