[llvm-branch-commits] [llvm] [AMDGPU] Measure MFMA hazard windows at their own producers (PR #225989)
Brendon Cahoon via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 5 06:46:30 PDT 2026
https://github.com/bcahoon updated https://github.com/llvm/llvm-project/pull/225989
>From 5fc97e129c245db1577d8c4e1aee613d0f286e63 Mon Sep 17 00:00:00 2001
From: Michael Halkenhaeuser <MichaelGerald.Halkenhauser at amd.com>
Date: Wed, 23 Sep 2026 13:07:08 -0500
Subject: [PATCH 1/3] [AMDGPU] Measure MFMA read hazards at each producer
Introduce more sophisticated traversal to avoid the following traps:
- order-dependent traversal and discarding seen BBs despite shorter path
- mis-matching distance and window of different producers
Record the best distance per BB instead of a visited flag and sweep the
arrivals in nondecreasing distance (bucket queue). This pairs producers
with their actual distance to a consumer in one go.
Fixed scenarios:
- MFMA reading an MFMA result as srcA, srcB or srcC
- VALU, memory or export instruction reading an MFMA result
AI-assisted.
---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 284 +++++---
llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h | 10 +
.../CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 623 ++++++++++++++++++
.../CodeGen/AMDGPU/mai-hazards-gfx942.mir | 78 +++
4 files changed, 918 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 4d25a723ae4a7..c7b44d128541d 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -956,6 +956,119 @@ hasHazard(StateT InitialState,
return false;
}
+// Number of trailing terminators of \p Pred not executed on the edge to \p MBB:
+// control leaves at the first terminator branching to \p MBB, so the ones after
+// it are dead on that edge. A fallthrough executes them all.
+static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred,
+ const MachineBasicBlock *MBB) {
+ const SIInstrInfo &TII =
+ *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo();
+ unsigned Unexecuted = 0, Passed = 0;
+ for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend();
+ I != E && I->isTerminator(); ++I, ++Passed) {
+ // Only an explicit MBB destination identifies this edge; the last match is
+ // the earliest such branch in program order.
+ if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(0).isMBB() &&
+ TII.getBranchDestBlock(*I) == MBB)
+ Unexecuted = Passed;
+ }
+ return Unexecuted;
+}
+
+// The block walk behind GCNHazardRecognizer::getMaxWindowDeficit, which
+// documents what it returns. Can only be run in a hazard recognizer mode.
+static int getMaxWindowDeficit(
+ function_ref<std::optional<int>(const MachineInstr &)> WindowFor,
+ const MachineInstr *MI, int MaxWindow) {
+ int Deficit = 0;
+
+ // Arrivals are swept in nondecreasing distance, so the first one at a block
+ // that is not stale is the shortest way into it. A queued Start points past
+ // the block's terminators, with the ones the edge executes already counted
+ // in Distance.
+ struct Arrival {
+ const MachineBasicBlock *MBB;
+ MachineBasicBlock::const_reverse_instr_iterator Start;
+ int Distance;
+ };
+ SmallVector<SmallVector<Arrival, 2>, 4> Pending;
+ DenseMap<const MachineBasicBlock *, int> Best;
+
+ auto Arrive = [&](const MachineBasicBlock *MBB,
+ MachineBasicBlock::const_reverse_instr_iterator Start,
+ int Distance) {
+ // Nothing this far back can ask for more than the answer already holds.
+ if (MaxWindow - Distance <= Deficit)
+ return;
+
+ if (Pending.empty())
+ Pending.resize(MaxWindow);
+
+ auto [It, Inserted] = Best.try_emplace(MBB, Distance);
+ if (!Inserted) {
+ if (It->second <= Distance)
+ return;
+ It->second = Distance;
+ }
+
+ assert(Distance < MaxWindow && "arrival outside the window");
+ Pending[Distance].push_back({MBB, Start, Distance});
+ };
+
+ // Taken by value: Arrive can grow the bucket this arrival is stored in.
+ auto Scan = [&](Arrival A) {
+ if (Best.lookup(A.MBB) < A.Distance)
+ return;
+
+ int Distance = A.Distance;
+ for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) {
+ if (I->isBundle())
+ continue;
+
+ if (std::optional<int> Window = WindowFor(*I)) {
+ assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
+ Deficit = std::max(Deficit, *Window - Distance);
+ }
+
+ if (I->isInlineAsm())
+ continue;
+
+ Distance += SIInstrInfo::getNumWaitStates(*I);
+
+ if (Distance >= MaxWindow)
+ return;
+ }
+
+ for (const MachineBasicBlock *Pred : A.MBB->predecessors()) {
+ auto Start = Pred->instr_rbegin(), End = Pred->instr_rend();
+ for (unsigned N = countUnexecutedTerminators(*Pred, A.MBB); N; --N)
+ ++Start;
+
+ int Executed = 0;
+ for (; Start != End && Start->isTerminator(); ++Start) {
+ assert(!WindowFor(*Start) && "terminators must not ask for a window");
+ Executed += SIInstrInfo::getNumWaitStates(*Start);
+ }
+
+ Arrive(Pred, Start, Distance + Executed);
+ }
+ };
+
+ // Not recorded in Best: a backedge may rescan this block from its end.
+ Scan({MI->getParent(), std::next(MI->getReverseIterator()), 0});
+
+ // Arrivals are never queued below the distance being swept, so one increasing
+ // sweep suffices; equal distances land in the bucket in hand, which the inner
+ // loop picks up by re-reading its size rather than iterating. Best keeps a
+ // zero-cost cycle from looping.
+ for (int Distance = 0; Distance < static_cast<int>(Pending.size());
+ ++Distance)
+ for (unsigned I = 0; I != Pending[Distance].size(); ++I)
+ Scan(Pending[Distance][I]);
+
+ return Deficit;
+}
+
// Returns a minimum wait states since \p I walking all predecessors.
// Only scans until \p IsExpired does not return true.
// Can only be run in a hazard recognizer mode.
@@ -1010,6 +1123,33 @@ getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
Visited, GetNumWaitStates);
}
+int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow,
+ WindowForFn WindowFor) const {
+ if (isHazardRecognizerMode())
+ return ::getMaxWindowDeficit(WindowFor, CurrCycleInstr, MaxWindow);
+
+ // EmittedInstrs is capped and can be shorter than the widest window, which
+ // costs scheduling quality only: the standalone pass still pads.
+ int Deficit = 0, Distance = 0;
+ for (MachineInstr *MI : EmittedInstrs) {
+ if (MI) {
+ if (std::optional<int> Window = WindowFor(*MI)) {
+ assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
+ Deficit = std::max(Deficit, *Window - Distance);
+ }
+
+ if (MI->isInlineAsm())
+ continue;
+ }
+
+ Distance += MI ? SIInstrInfo::getNumWaitStates(*MI) : 1;
+
+ if (Distance >= MaxWindow)
+ break;
+ }
+ return Deficit;
+}
+
int GCNHazardRecognizer::getWaitStatesSince(
IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
if (isHazardRecognizerMode()) {
@@ -3410,35 +3550,21 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
if (!Use.isReg())
continue;
Register Reg = Use.getReg();
- const MachineInstr *MI1;
-
- auto IsOverlappedMFMAFn = [Reg, &MI1, this](const MachineInstr &MI) {
- if (!SIInstrInfo::isMFMA(MI))
- return false;
- Register DstReg = MI.getOperand(0).getReg();
- MI1 = &MI;
- return TRI.regsOverlap(DstReg, Reg);
- };
WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
- int NumWaitStates =
- getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
- if (NumWaitStates == std::numeric_limits<int>::max())
- continue;
-
int OpNo = Use.getOperandNo();
- int NeedWaitStates = getMFMAReadWaitStates(*MI, *MI1, Reg, OpNo == SrcCIdx);
-
- assert(NeedWaitStates <= MaxWaitStates &&
- "hazard requirement exceeds the scan window");
- if (WaitStatesNeeded >= NeedWaitStates)
- continue;
-
- WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
- WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+ WaitStatesNeeded = std::max(
+ WaitStatesNeeded,
+ getMaxWindowDeficit(
+ MaxWaitStates, [&](const MachineInstr &P) -> std::optional<int> {
+ if (!SIInstrInfo::isMFMA(P) ||
+ !TRI.regsOverlap(P.getOperand(0).getReg(), Reg))
+ return std::nullopt;
+ return getMFMAReadWaitStates(*MI, P, Reg, OpNo == SrcCIdx);
+ }));
if (WaitStatesNeeded == MaxWaitStates)
break;
@@ -3594,8 +3720,8 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
- const MachineInstr *MFMA = nullptr;
unsigned Reg;
+ const MachineInstr *MFMA = nullptr;
auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
if (!SIInstrInfo::isMFMA(MI) ||
!TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
@@ -3683,63 +3809,67 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
}
}
- MFMA = nullptr;
- WaitStatesSinceDef =
- getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
- if (!MFMA)
- continue;
-
- unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
- int NumPasses = HazardDefLatency;
- int NeedWaitStates = MaxWaitStates;
-
- if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
- switch (HazardDefLatency) {
- case 4:
- NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
- : DMFMA4x4WriteVgprVALUReadWaitStates;
- break;
- case 8:
- case 16:
+ auto MFMAWindow = [&](const MachineInstr &Producer) {
+ int NumPasses = TSchedModel.computeInstrLatency(&Producer);
+ int NeedWaitStates = MaxWaitStates;
+
+ if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
+ switch (NumPasses) {
+ case 4:
+ NeedWaitStates = IsMemOrExport
+ ? DMFMA4x4WriteVgprMemExpReadWaitStates
+ : DMFMA4x4WriteVgprVALUReadWaitStates;
+ break;
+ case 8:
+ case 16:
+ NeedWaitStates =
+ IsMemOrExport
+ ? DMFMA16x16WriteVgprMemExpReadWaitStates
+ : (ST.hasGFX950Insts()
+ ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
+ : DMFMA16x16WriteVgprVALUReadWaitStates);
+ break;
+ default:
+ llvm_unreachable("unexpected dgemm");
+ }
+ } else if (ST.hasGFX940Insts()) {
NeedWaitStates =
- IsMemOrExport
- ? DMFMA16x16WriteVgprMemExpReadWaitStates
- : (ST.hasGFX950Insts()
- ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
- : DMFMA16x16WriteVgprVALUReadWaitStates);
- break;
- default:
- llvm_unreachable("unexpected dgemm");
- }
- } else if (ST.hasGFX940Insts()) {
- NeedWaitStates =
- TII.isXDL(*MFMA)
- ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
- NumPasses, ST.hasGFX950Insts())
- : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
- NumPasses);
- } else {
- switch (HazardDefLatency) {
- case 2:
- NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
- break;
- case 8:
- NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
- break;
- case 16:
- NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
- break;
- default:
- llvm_unreachable("unexpected number of passes for mfma");
+ TII.isXDL(Producer)
+ ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
+ NumPasses, ST.hasGFX950Insts())
+ : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
+ NumPasses);
+ } else {
+ switch (NumPasses) {
+ case 2:
+ NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
+ break;
+ case 8:
+ NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
+ break;
+ case 16:
+ NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
+ break;
+ default:
+ llvm_unreachable("unexpected number of passes for mfma");
+ }
}
- }
- assert(NeedWaitStates <= MaxWaitStates &&
- "hazard requirement exceeds the scan window");
- int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
- WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+ assert(NeedWaitStates <= MaxWaitStates &&
+ "hazard requirement exceeds the scan window");
+ return NeedWaitStates;
+ };
- if (WaitStatesNeeded == MaxWaitStates)
+ WaitStatesNeeded = std::max(
+ WaitStatesNeeded,
+ getMaxWindowDeficit(MaxWaitStates,
+ [&](const MachineInstr &P) -> std::optional<int> {
+ if (!IsMFMAWriteFn(P))
+ return std::nullopt;
+ return MFMAWindow(P);
+ }));
+
+ if (WaitStatesNeeded >= MaxWaitStates)
break;
}
}
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index 60ebb73f99286..3910f676c9e98 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -176,6 +176,16 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
// used on a newly inserted instruction before returning from PreEmitNoops.
void runOnInstruction(MachineInstr *MI);
+ /// Wait states required after \p MI, or nullopt if \p MI is not waited for.
+ /// Must return nullopt for terminators.
+ typedef function_ref<std::optional<int>(const MachineInstr &)> WindowForFn;
+
+ /// Returns the largest WindowFor(I) - distance(I) over the instructions
+ /// preceding the one being checked within \p MaxWindow, which must bound
+ /// every window WindowFor can return, and zero if it accepts none. Each
+ /// window is paired with the distance to the instruction that supplied it.
+ int getMaxWindowDeficit(int MaxWindow, WindowForFn WindowFor) const;
+
int getWaitStatesSince(IsHazardFn IsHazard, int Limit,
GetNumWaitStatesFn GetNumWaitStates) const;
int getWaitStatesSince(IsHazardFn IsHazard, int Limit) const;
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
index bf11056022b7d..f62457aa5f3d4 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
@@ -1453,3 +1453,626 @@ body: |
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X4F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_I32_16X16X4I8_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
...
+# Window 11 with the nearest producer 1 wait state away needs 10; taking the
+# detour's distance leaves it two short.
+# GCN-LABEL: name: smfma16x16_write_vgpr_valu_read_diamond_long_edge_first
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2
+name: smfma16x16_write_vgpr_valu_read_diamond_long_edge_first
+body: |
+ bb.0:
+ successors: %bb.2
+ S_BRANCH %bb.2
+
+ bb.1:
+ successors: %bb.3
+ $sgpr0 = S_MOV_B32 0
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.1, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.3:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+...
+# GCN-LABEL: name: smfma16x16_write_vgpr_valu_read_diamond_short_edge_first
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2
+name: smfma16x16_write_vgpr_valu_read_diamond_short_edge_first
+body: |
+ bb.0:
+ successors: %bb.1
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+
+ bb.2:
+ successors: %bb.3
+ $sgpr0 = S_MOV_B32 0
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+...
+
+# The same with a detour longer than the window: expiring on the long edge must
+# not stop the direct edge from being measured.
+
+# GCN-LABEL: name: smfma16x16_valu_read_detour_longer_than_window
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2
+name: smfma16x16_valu_read_detour_longer_than_window
+body: |
+ bb.0:
+ successors: %bb.2
+ S_BRANCH %bb.2
+
+ bb.1:
+ successors: %bb.3
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ $sgpr3 = S_MOV_B32 0
+ $sgpr4 = S_MOV_B32 0
+ $sgpr5 = S_MOV_B32 0
+ $sgpr6 = S_MOV_B32 0
+ $sgpr7 = S_MOV_B32 0
+ $sgpr8 = S_MOV_B32 0
+ $sgpr9 = S_MOV_B32 0
+ $sgpr10 = S_MOV_B32 0
+ $sgpr11 = S_MOV_B32 0
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.1, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.3:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+...
+# An MFMA overlapping nothing must not set the window for one that does: the
+# 32x32 writing v[0:3] asks for 16, the unrelated 4x4 for 2.
+# GCN-LABEL: name: smfma32x32_srcc_read_unrelated_mfma_producer_block_first
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 14
+# GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32
+name: smfma32x32_srcc_read_unrelated_mfma_producer_block_first
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# GCN-LABEL: name: smfma32x32_srcc_read_unrelated_mfma_decoy_block_first
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 14
+# GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32
+name: smfma32x32_srcc_read_unrelated_mfma_decoy_block_first
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# srcC is exactly the producer's destination, so nothing is needed; an unrelated
+# MFMA of the same shape must not turn that into 15.
+# GCN-LABEL: name: smfma32x32_full_srcc_read_unrelated_mfma
+# GCN: bb.3:
+# GCN-NEXT: early-clobber {{.*}} = V_MFMA_F32_32X32X2F32
+name: smfma32x32_full_srcc_read_unrelated_mfma
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# Window and distance must come from one producer: the 32x32 needs 19 from 11
+# away, the 4x4 needs 5 from 1 away, so 8. Mispairing would ask 18.
+
+# GCN-LABEL: name: mfma_mixed_window_far_wide_near_narrow
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 7
+# GCN-NEXT: $vgpr16 = V_MOV_B32
+name: mfma_mixed_window_far_wide_near_narrow
+body: |
+ bb.0:
+ successors: %bb.2, %bb.1
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ $sgpr3 = S_MOV_B32 0
+ $sgpr4 = S_MOV_B32 0
+ $sgpr5 = S_MOV_B32 0
+ $sgpr6 = S_MOV_B32 0
+ $sgpr7 = S_MOV_B32 0
+ $sgpr8 = S_MOV_B32 0
+ $sgpr9 = S_MOV_B32 0
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_ENDPGM 0
+...
+# The same around a loop: the 16x16 needs 11 from 1 away, the 4x4 needs 5 around
+# the latch, so 10. Mispairing would ask 4.
+
+# GCN-LABEL: name: mfma_mixed_window_loop
+# GCN: bb.2:
+# GCN: S_NOP 9
+# GCN-NEXT: $vgpr16 = V_MOV_B32
+name: mfma_mixed_window_loop
+body: |
+ bb.0:
+ successors: %bb.1
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3, %bb.4
+ $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_CBRANCH_SCC1 %bb.4, implicit $scc
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.2
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.4:
+ S_ENDPGM 0
+...
+# A trailing branch to another successor is not executed on the edge into the
+# consumer, so it must not count. Both need 10.
+
+# GCN-LABEL: name: smfma16x16_valu_read_two_terminators
+# GCN: bb.2:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr40 = V_MOV_B32
+name: smfma16x16_valu_read_two_terminators
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ S_BRANCH %bb.2
+
+ bb.2:
+ $vgpr40 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_ENDPGM 0
+...
+# Only the bb.3 edge skips bb.0's trailing S_BRANCH, so the edges arrive at
+# different distances; along the cheaper one 3 of the 19 are left.
+
+# GCN-LABEL: name: smfma32x32_valu_read_unexecuted_branch_on_one_edge
+# GCN: bb.4:
+# GCN: $sgpr5 = S_MOV_B32 0
+# GCN-NEXT: S_NOP 2
+# GCN-NEXT: $vgpr16 = V_MOV_B32
+name: smfma32x32_valu_read_unexecuted_branch_on_one_edge
+body: |
+ bb.0:
+ successors: %bb.1, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+ S_BRANCH %bb.1
+ bb.1:
+ successors: %bb.2, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+ S_BRANCH %bb.2
+ bb.2:
+ successors: %bb.4
+ $sgpr0 = S_MOV_B32 0
+ S_BRANCH %bb.4
+ bb.3:
+ successors: %bb.4
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ $sgpr3 = S_MOV_B32 0
+ $sgpr4 = S_MOV_B32 0
+ S_BRANCH %bb.4
+ bb.4:
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ $sgpr3 = S_MOV_B32 0
+ $sgpr4 = S_MOV_B32 0
+ $sgpr5 = S_MOV_B32 0
+ $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_ENDPGM 0
+...
+# The same when the reader is an MFMA: srcA after a 32x32 needs 19, after a 4x4
+# needs 5, one branch away each, so 18.
+
+# GCN-LABEL: name: smfma_srca_read_mixed_shapes
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 15
+# GCN-NEXT: S_NOP 1
+# GCN-NEXT: $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32
+name: smfma_srca_read_mixed_shapes
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $agpr1, $vgpr0, $agpr20_agpr21_agpr22_agpr23, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# A producer writing exactly the srcC register needs nothing and must not hide a
+# farther partial overlap: the 16x16 needs 8 from 1 away.
+
+# GCN-LABEL: name: smfma_full_srcc_producer_does_not_hide_partial
+# GCN: V_MFMA_F32_4X4X1F32
+# GCN-NEXT: S_NOP 6
+# GCN-NEXT: $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32
+name: smfma_full_srcc_producer_does_not_hide_partial
+body: |
+ bb.0:
+ $agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17, 0, 0, 0, implicit $mode, implicit $exec
+ $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr30_agpr31_agpr32_agpr33, 0, 0, 0, implicit $mode, implicit $exec
+ $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# Control leaves at the first terminator that transfers to the successor, so a
+# taken conditional branch is the only one executed: 1 of the 11.
+
+# GCN-LABEL: name: smfma16x16_valu_read_same_target_terminators
+# GCN: bb.1:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr4 = V_MOV_B32
+name: smfma16x16_valu_read_same_target_terminators
+body: |
+ bb.0:
+ successors: %bb.1
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+...
+# Each successor executes a different number of the three terminators, so the
+# same producer is 1 and 3 wait states away.
+
+# GCN-LABEL: name: smfma16x16_valu_read_first_conditional_target
+# GCN: bb.2:
+# GCN-NEXT: S_NOP 9
+# GCN-NEXT: $vgpr4 = V_MOV_B32
+name: smfma16x16_valu_read_first_conditional_target
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_CBRANCH_EXECZ %bb.3, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ S_ENDPGM 0
+
+ bb.2:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+
+ bb.3:
+ S_ENDPGM 0
+...
+# GCN-LABEL: name: smfma16x16_valu_read_unconditional_target
+# GCN: bb.1:
+# GCN-NEXT: S_NOP 7
+# GCN-NEXT: $vgpr4 = V_MOV_B32
+name: smfma16x16_valu_read_unconditional_target
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_CBRANCH_EXECZ %bb.3, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_ENDPGM 0
+
+ bb.2:
+ S_ENDPGM 0
+
+ bb.3:
+ S_ENDPGM 0
+...
+
+# The empty fallthrough contributes nothing: along bb.0 -> bb.1 -> bb.2 only the
+# conditional branch executes, leaving 5 of the 11.
+
+# GCN-LABEL: name: smfma16x16_valu_read_across_empty_fallthrough
+# GCN: bb.2:
+# GCN-NEXT: $sgpr0 = S_MOV_B32 0
+# GCN: S_NOP 4
+# GCN-NEXT: $vgpr4 = V_MOV_B32
+name: smfma16x16_valu_read_across_empty_fallthrough
+body: |
+ bb.0:
+ successors: %bb.1, %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.3
+
+ bb.1:
+ successors: %bb.2
+
+ bb.2:
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 0
+ $sgpr2 = S_MOV_B32 0
+ $sgpr3 = S_MOV_B32 0
+ $sgpr4 = S_MOV_B32 0
+ $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_ENDPGM 0
+
+ bb.3:
+ S_ENDPGM 0
+...
+
+# Blocks holding nothing but a branch: with no instruction to notice the window
+# is spent, the bound has to be tested on the edge. 24 is past it.
+
+# GCN-LABEL: name: valu_read_across_long_branch_chain
+# GCN: bb.24:
+# GCN-NEXT: $vgpr8 = V_MOV_B32
+name: valu_read_across_long_branch_chain
+body: |
+ bb.0:
+ successors: %bb.1
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr1, $vgpr0, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.5
+ S_BRANCH %bb.5
+
+ bb.5:
+ successors: %bb.6
+ S_BRANCH %bb.6
+
+ bb.6:
+ successors: %bb.7
+ S_BRANCH %bb.7
+
+ bb.7:
+ successors: %bb.8
+ S_BRANCH %bb.8
+
+ bb.8:
+ successors: %bb.9
+ S_BRANCH %bb.9
+
+ bb.9:
+ successors: %bb.10
+ S_BRANCH %bb.10
+
+ bb.10:
+ successors: %bb.11
+ S_BRANCH %bb.11
+
+ bb.11:
+ successors: %bb.12
+ S_BRANCH %bb.12
+
+ bb.12:
+ successors: %bb.13
+ S_BRANCH %bb.13
+
+ bb.13:
+ successors: %bb.14
+ S_BRANCH %bb.14
+
+ bb.14:
+ successors: %bb.15
+ S_BRANCH %bb.15
+
+ bb.15:
+ successors: %bb.16
+ S_BRANCH %bb.16
+
+ bb.16:
+ successors: %bb.17
+ S_BRANCH %bb.17
+
+ bb.17:
+ successors: %bb.18
+ S_BRANCH %bb.18
+
+ bb.18:
+ successors: %bb.19
+ S_BRANCH %bb.19
+
+ bb.19:
+ successors: %bb.20
+ S_BRANCH %bb.20
+
+ bb.20:
+ successors: %bb.21
+ S_BRANCH %bb.21
+
+ bb.21:
+ successors: %bb.22
+ S_BRANCH %bb.22
+
+ bb.22:
+ successors: %bb.23
+ S_BRANCH %bb.23
+
+ bb.23:
+ successors: %bb.24
+ S_BRANCH %bb.24
+
+ bb.24:
+ $vgpr8 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ S_ENDPGM 0
+...
+# Every block in the empty chain is reached at the same distance, once by
+# fallthrough and once from a branch below, so the traversal keeps queueing
+# arrivals at the distance it is currently visiting. The chain costs nothing,
+# leaving the whole 11 of the window.
+
+# GCN-LABEL: name: smfma16x16_valu_read_many_equal_distance_arrivals
+# GCN: bb.7:
+# GCN: S_NOP 10
+# GCN-NEXT: $vgpr4 = V_MOV_B32
+name: smfma16x16_valu_read_many_equal_distance_arrivals
+body: |
+ bb.0:
+ successors: %bb.1
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.1:
+ successors: %bb.2
+
+ bb.2:
+ successors: %bb.3
+
+ bb.3:
+ successors: %bb.4
+
+ bb.4:
+ successors: %bb.5
+
+ bb.5:
+ successors: %bb.6
+
+ bb.6:
+ successors: %bb.7
+
+ bb.7:
+ successors: %bb.8
+ $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec
+ S_BRANCH %bb.8
+
+ bb.8:
+ successors: %bb.1, %bb.9
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.9:
+ successors: %bb.2, %bb.10
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.10:
+ successors: %bb.3, %bb.11
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+
+ bb.11:
+ successors: %bb.4, %bb.12
+ S_CBRANCH_SCC1 %bb.4, implicit $scc
+
+ bb.12:
+ successors: %bb.5, %bb.13
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.13:
+ successors: %bb.6
+ S_CBRANCH_SCC1 %bb.6, implicit $scc
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
index 2f78efbdd256e..f97df59ee1b52 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
@@ -2715,3 +2715,81 @@ body: |
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
...
+
+# Window and distance must come from one producer: srcA after a 16-pass XDL
+# needs NumPasses + 3, one more on gfx950, after a 2-pass non-XDL NumPasses + 2.
+
+# GCN-LABEL: name: mfma_srca_read_mixed_xdl_and_smfma
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 15
+# GFX942-NEXT: S_NOP 1
+# GFX950-NEXT: S_NOP 2
+# GCN-NEXT: $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32
+name: mfma_srca_read_mixed_xdl_and_smfma
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr38, $vgpr39, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr39, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
+# The same for a srcC read, whose windows come from another table: 8-pass XDL
+# needs NumPasses + 1, one more on gfx950, the 2-pass non-XDL one 2.
+
+# GCN-LABEL: name: mfma_srcc_read_mixed_xdl_and_smfma
+# GCN: bb.3:
+# GFX942-NEXT: S_NOP 7
+# GFX950-NEXT: S_NOP 8
+# GCN-NEXT: $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32
+name: mfma_srcc_read_mixed_xdl_and_smfma
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+# A 4-pass producer writing exactly the srcC register asks nothing and must not
+# end the search: the 8-pass XDL result 2 away needs 9, 10 on gfx950.
+
+# GCN-LABEL: name: mfma_full_srcc_producer_does_not_hide_partial
+# GCN: V_MFMA_F32_16X16X16F16
+# GFX942-NEXT: S_NOP 7
+# GFX950-NEXT: S_NOP 8
+# GCN-NEXT: $agpr40_agpr41_agpr42_agpr43 = V_MFMA_F32_4X4X1F32
+name: mfma_full_srcc_producer_does_not_hide_partial
+body: |
+ bb.0:
+ $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+ $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_16X16X16F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr30_agpr31_agpr32_agpr33, 0, 0, 0, implicit $mode, implicit $exec
+ $agpr40_agpr41_agpr42_agpr43 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
>From a122e864583bc42fa1e88d54f40eb012de6e69a7 Mon Sep 17 00:00:00 2001
From: Michael Halkenhaeuser <MichaelGerald.Halkenhauser at amd.com>
Date: Wed, 23 Sep 2026 12:52:16 -0500
Subject: [PATCH 2/3] [AMDGPU] Measure MFMA overwrite hazards at each
instruction
Apply previously established processing to:
- VALU overwriting an MFMA result
- VALU overwriting a register an MFMA took as srcC
AI-assisted.
---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 96 +++++++++----------
.../CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 51 ++++++++++
.../CodeGen/AMDGPU/mai-hazards-gfx942.mir | 58 +++++++++++
3 files changed, 156 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index c7b44d128541d..0c13255a68e72 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1017,6 +1017,7 @@ static int getMaxWindowDeficit(
// Taken by value: Arrive can grow the bucket this arrival is stored in.
auto Scan = [&](Arrival A) {
+ // A closer arrival has superseded this one.
if (Best.lookup(A.MBB) < A.Distance)
return;
@@ -3721,12 +3722,10 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
unsigned Reg;
- const MachineInstr *MFMA = nullptr;
- auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
+ auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) {
if (!SIInstrInfo::isMFMA(MI) ||
!TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
return false;
- MFMA = &MI;
return true;
};
@@ -3912,14 +3911,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
WaitStatesSinceDef);
- MFMA = nullptr;
- WaitStatesSinceDef =
- getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
- if (MFMA) {
+ auto MFMAWawWindow = [&](const MachineInstr &Producer) {
int NeedWaitStates = MaxWaitStates;
- int NumPasses = TSchedModel.computeInstrLatency(MFMA);
+ int NumPasses = TSchedModel.computeInstrLatency(&Producer);
- if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
+ if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
switch (NumPasses) {
case 4:
NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
@@ -3933,7 +3929,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
}
} else if (ST.hasGFX940Insts()) {
NeedWaitStates =
- TII.isXDL(*MFMA)
+ TII.isXDL(Producer)
? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
@@ -3955,54 +3951,56 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
assert(NeedWaitStates <= MaxWaitStates &&
"hazard requirement exceeds the scan window");
- int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
- WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+ return NeedWaitStates;
+ };
- if (WaitStatesNeeded == MaxWaitStates)
- break;
- }
+ WaitStatesNeeded = std::max(
+ WaitStatesNeeded,
+ getMaxWindowDeficit(MaxWaitStates,
+ [&](const MachineInstr &P) -> std::optional<int> {
+ if (!IsMFMAWriteFn(P))
+ return std::nullopt;
+ return MFMAWawWindow(P);
+ }));
+
+ if (WaitStatesNeeded >= MaxWaitStates)
+ break;
- auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
- if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) ||
- !MI.readsRegister(Reg, &TRI))
- return false;
+ // Wait states this write needs after an MFMA that read Reg as srcC.
+ auto SMFMAReadAsCWindow =
+ [&](const MachineInstr &Reader) -> std::optional<int> {
+ if (!SIInstrInfo::isMFMA(Reader) ||
+ SIInstrInfo::isDGEMM(Reader.getOpcode()) ||
+ !Reader.readsRegister(Reg, &TRI))
+ return std::nullopt;
- if (ST.hasGFX940Insts() && !TII.isXDL(MI))
- return false;
+ if (ST.hasGFX940Insts() && !TII.isXDL(Reader))
+ return std::nullopt;
const MachineOperand *SrcC =
- TII.getNamedOperand(MI, AMDGPU::OpName::src2);
- assert(SrcC);
+ TII.getNamedOperand(Reader, AMDGPU::OpName::src2);
+ assert(SrcC && "an MFMA must have a src2 operand");
if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
- return false;
+ return std::nullopt;
- MFMA = &MI;
- return true;
+ switch (TSchedModel.computeInstrLatency(&Reader)) {
+ case 2:
+ return SMFMA4x4ReadVgprVALUWarWaitStates;
+ case 4:
+ assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later");
+ return GFX940_XDL4PassReadVgprVALUWarWaitStates;
+ case 8:
+ return SMFMA16x16ReadVgprVALUWarWaitStates;
+ case 16:
+ [[fallthrough]];
+ default:
+ return SMFMA32x32ReadVgprVALUWarWaitStates;
+ }
};
- MFMA = nullptr;
- int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
- MaxWarWaitStates);
- if (!MFMA)
- continue;
-
- unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
- int NeedWaitStates = MaxWaitStates;
- switch (HazardDefLatency) {
- case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
- break;
- case 4: assert(ST.hasGFX940Insts());
- NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
- break;
- case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
- break;
- case 16: [[fallthrough]];
- default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
- break;
- }
-
- int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
- WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+ WaitStatesNeeded =
+ std::max(WaitStatesNeeded,
+ getMaxWindowDeficit(MaxWarWaitStates, SMFMAReadAsCWindow));
}
return WaitStatesNeeded;
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
index f62457aa5f3d4..340b366fc8802 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
@@ -1872,6 +1872,57 @@ body: |
S_ENDPGM 0
...
+# A VALU overwrite takes the window of the producer on its own path: 16 passes
+# need 19, 2 passes need 5, one branch away each, so 18.
+
+# GCN-LABEL: name: smfma_valu_write_mixed_shapes
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 15
+# GCN-NEXT: S_NOP 1
+# GCN-NEXT: $vgpr0 = V_MOV_B32
+name: smfma_valu_write_mixed_shapes
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+ bb.2:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+ bb.3:
+ $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ S_ENDPGM 0
+...
+# The same for a register read as srcC: 15 after a 32x32 read it, 7 after a
+# 16x16, one branch away each, so 14.
+
+# GCN-LABEL: name: smfma_srcc_read_valu_write_mixed_shapes
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 13
+# GCN-NEXT: $vgpr0 = V_MOV_B32
+name: smfma_srcc_read_valu_write_mixed_shapes
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+ bb.1:
+ successors: %bb.3
+ $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+ bb.2:
+ successors: %bb.3
+ $vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+ bb.3:
+ $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ S_ENDPGM 0
+...
# The empty fallthrough contributes nothing: along bb.0 -> bb.1 -> bb.2 only the
# conditional branch executes, leaving 5 of the 11.
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
index f97df59ee1b52..f3d0e857b3777 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
@@ -2793,3 +2793,61 @@ body: |
S_ENDPGM 0
...
+# A VALU overwrite takes the window of the producer on its path: 16-pass XDL
+# needs NumPasses + 3, one more on gfx950, against NumPasses + 2.
+
+# GCN-LABEL: name: waw_mixed_xdl_and_smfma
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 15
+# GFX942-NEXT: S_NOP 1
+# GFX950-NEXT: S_NOP 2
+# GCN-NEXT: $vgpr0 = V_MOV_B32
+name: waw_mixed_xdl_and_smfma
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr38, $vgpr39, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ S_ENDPGM 0
+...
+# The same where the register was read as srcC: 15 after a 16-pass reader, 3
+# after a 4-pass one, so 14. The 4-pass arm is gfx940 and later only.
+
+# GCN-LABEL: name: war_mixed_xdl_shapes
+# GCN: bb.3:
+# GCN-NEXT: S_NOP 13
+# GCN-NEXT: $vgpr0 = V_MOV_B32
+name: war_mixed_xdl_shapes
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.3
+ $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ $vgpr96_vgpr97_vgpr98_vgpr99 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ S_ENDPGM 0
+...
>From 58aca82e97049b40bc94bc95d178f5404c258599 Mon Sep 17 00:00:00 2001
From: Michael Halkenhaeuser <MichaelGerald.Halkenhauser at amd.com>
Date: Mon, 28 Sep 2026 08:30:25 -0500
Subject: [PATCH 3/3] fixup! [AMDGPU] Measure MFMA read hazards at each
producer
Reword the comments of the two tests where an exact srcC write must not
hide an earlier partial one: name the producers, give the nearer one's
window, and keep the farther one's window apart from the padding it
leaves. On gfx940+ the exact write now has a window of its own, since
two different MFMAs sharing an accumulator need wait states.
AI-assisted.
---
llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 4 ++--
llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
index 340b366fc8802..f83cb5b9d940d 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir
@@ -1788,8 +1788,8 @@ body: |
$agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $agpr1, $vgpr0, $agpr20_agpr21_agpr22_agpr23, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
-# A producer writing exactly the srcC register needs nothing and must not hide a
-# farther partial overlap: the 16x16 needs 8 from 1 away.
+# The first 4x4 writes exactly the srcC register, with no window, but must not
+# hide the 16x16 one instruction back: its window of 8 leaves 7 to pad.
# GCN-LABEL: name: smfma_full_srcc_producer_does_not_hide_partial
# GCN: V_MFMA_F32_4X4X1F32
diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
index f3d0e857b3777..cc272b5cc98ea 100644
--- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir
@@ -2776,8 +2776,8 @@ body: |
$agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
-# A 4-pass producer writing exactly the srcC register asks nothing and must not
-# end the search: the 8-pass XDL result 2 away needs 9, 10 on gfx950.
+# The 16x16 writes exactly the srcC register, with a window of 5 (6 on gfx950),
+# but must not hide the 32x32 with a window of 9 (10) and leaves 8 (9) to pad.
# GCN-LABEL: name: mfma_full_srcc_producer_does_not_hide_partial
# GCN: V_MFMA_F32_16X16X16F16
More information about the llvm-branch-commits
mailing list