[llvm-branch-commits] [llvm] [AMDGPU] Allow MachinePipeliner to consider loops with large MIIs (PR #212537)

[via llvm-branch-commits]

https://github.com/hjagasiaAMD updated https://github.com/llvm/llvm-project/pull/212537

>From 1c3c31110817e90f69900ddd9bdf06aad0c30709 Mon Sep 17 00:00:00 2001
From: Harsha Jagasia <harsha.jagasia at amd.com>
Date: Tue, 11 Aug 2026 14:55:30 -0500
Subject: [PATCH] [MachinePipeliner] Let targets configure the maximum MII

The pipeliner rejects a loop whose minimum initiation interval exceeds
-pipeliner-max-mii. Its default of 27 suits targets with short instruction
latencies, but is too small for others: on AMDGPU a couple of MFMA
instructions already push the MII past it, so the loop never pipelines.

Move the limit into a MachinePipelinerPolicy that targets customize by
implementing TargetSubtargetInfo::overridePipelinerPolicy(). The generic
default is unchanged and an explicit -pipeliner-max-mii still wins over the
target's choice, so no target changes behavior here. Exercised by the AMDGPU
adoption in a following commit.
---
 llvm/include/llvm/CodeGen/MachinePipeliner.h    | 14 ++++++++++++++
 llvm/include/llvm/CodeGen/TargetSubtargetInfo.h |  4 ++++
 llvm/lib/CodeGen/MachinePipeliner.cpp           | 15 ++++++++++++---
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 19699f069ae72..41ab7c6891bb1 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -63,6 +63,14 @@ class SMSchedule;
 extern LLVM_ABI cl::opt<bool> SwpEnableCopyToPhi;
 extern LLVM_ABI cl::opt<int> SwpForceIssueWidth;
 
+/// Software pipelining policy for a loop, which a target can customize by
+/// implementing TargetSubtargetInfo::overridePipelinerPolicy.
+struct MachinePipelinerPolicy {
+  /// Don't pipeline loops whose minimum initiation interval exceeds this.
+  /// Overridden by -pipeliner-max-mii when that is passed.
+  int MaxMII = 27;
+};
+
 /// The main class in the implementation of the target independent
 /// software pipeliner pass.
 class LLVM_ABI MachinePipeliner : public MachineFunctionPass {
@@ -296,6 +304,9 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
   unsigned II_setByPragma = 0;
   TargetInstrInfo::PipelinerLoopInfo *LoopPipelinerInfo = nullptr;
 
+  /// Policy for this loop, after target and command line overrides.
+  MachinePipelinerPolicy Policy;
+
   /// A topological ordering of the SUnits, which is needed for changing
   /// dependences and iterating over the SUnits.
   ScheduleDAGTopologicalSort Topo;
@@ -387,6 +398,7 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
       : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis),
         RegClassInfo(rci), II_setByPragma(II), LoopPipelinerInfo(PLI),
         Topo(SUnits, &ExitSU), AA(AA), BAA(*AA) {
+    initPolicy();
     P.MF->getSubtarget().getSMSMutations(Mutations);
     if (SwpEnableCopyToPhi)
       Mutations.push_back(std::make_unique<CopyToPhiMutation>());
@@ -453,6 +465,8 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
                              const MachineInstr *OtherMI) const;
 
 private:
+  /// Set the policy for this loop, allowing the target to override it.
+  void initPolicy();
   LoopCarriedEdges addLoopCarriedDependences();
   void updatePhiDependences();
   void changeDependences();
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index a50f7f804b03d..616ed755de2be 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -42,6 +42,7 @@ class InstructionSelector;
 class LegalizerInfo;
 class LibcallLoweringInfo;
 class MachineInstr;
+struct MachinePipelinerPolicy;
 struct MachineSchedPolicy;
 struct MCReadAdvanceEntry;
 struct MCSchedModel;
@@ -276,6 +277,9 @@ class LLVM_ABI TargetSubtargetInfo : public MCSubtargetInfo {
   virtual void overridePostRASchedPolicy(MachineSchedPolicy &Policy,
                                          const SchedRegion &Region) const {}
 
+  /// Override generic software pipelining policy.
+  virtual void overridePipelinerPolicy(MachinePipelinerPolicy &Policy) const {}
+
   // Perform target-specific adjustments to the latency of a schedule
   // dependency.
   // If a pair of operands is associated with the schedule dependency, DefOpIdx
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index a97e0702d1911..d24e2425c8e14 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -813,8 +813,8 @@ void SwingSchedulerDAG::schedule() {
   }
 
   // Don't pipeline large loops.
-  if (SwpMaxMii != -1 && (int)MII > SwpMaxMii) {
-    LLVM_DEBUG(dbgs() << "MII > " << SwpMaxMii
+  if (Policy.MaxMII != -1 && (int)MII > Policy.MaxMII) {
+    LLVM_DEBUG(dbgs() << "MII > " << Policy.MaxMII
                       << ", we don't pipeline large loops\n");
     NumFailLargeMaxMII++;
     Pass.ORE->emit([&]() {
@@ -822,7 +822,7 @@ void SwingSchedulerDAG::schedule() {
                  DEBUG_TYPE, "schedule", Loop.getStartLoc(), Loop.getHeader())
              << "Minimal Initiation Interval too large: "
              << ore::NV("MII", (int)MII) << " > "
-             << ore::NV("SwpMaxMii", SwpMaxMii) << "."
+             << ore::NV("SwpMaxMii", Policy.MaxMII) << "."
              << "Refer to -pipeliner-max-mii.";
     });
     return;
@@ -2782,6 +2782,15 @@ void SwingSchedulerDAG::computeNodeOrder(NodeSetType &NodeSets) {
   });
 }
 
+/// Set the policy for this loop, allowing the target to override it.
+void SwingSchedulerDAG::initPolicy() {
+  MF.getSubtarget().overridePipelinerPolicy(Policy);
+
+  // After subtarget overrides, apply command line options.
+  if (SwpMaxMii.getNumOccurrences())
+    Policy.MaxMII = SwpMaxMii;
+}
+
 /// Process the nodes in the computed order and create the pipelined schedule
 /// of the instructions, if possible. Return true if a schedule is found.
 bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {