[llvm] [AMDGPU] Factor MFMA-form copy analysis into shared copy-plan analysis (PR #209438)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 03:59:08 PDT 2026
https://github.com/xgxanq created https://github.com/llvm/llvm-project/pull/209438
None
>From b5505ef17623cf8e48c1d3d19018bbc8120039ad Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Fri, 3 Jul 2026 03:54:44 +0000
Subject: [PATCH 1/5] [AMDGPU] Add safe-guard exclusion for MFMA form rewrite
RewriteMFMAFormStage previously used a coarse isRewriteCandidate check
that rejected any MFMA whose dst had a non-MFMA/non-COPY user. Replace
that with a precise exclusion analysis that screens each candidate for
cases where reclassifying to AGPR form would be illegal, and keeps the
rest rewritable.
findReachingDefs is reworked into a SubRange-aware implementation backed
by collectReachingDefsInRange:
- Subreg uses query the SubRange whose LaneMask fully covers the operand
lanes, falling back to all overlapping SubRanges when no single
SubRange provides full coverage.
- Full-reg uses collect across all SubRanges and deduplicate via a
SmallSet (a full-width def appears in every SubRange but counts once).
- Self-referential defs (src2 == dst MFMA) are skipped in the traversal.
findReachingUses skips implicit operands so that a partial subreg def's
implicit full-reg use (RMW lane preservation) is not treated as a real
consumer, preventing spurious bridge copies.
New exclusion analysis (computeExclusionSet), run before any rewriting:
- hasSrc2BridgeConflict: detects when the src2 bridge-copy model breaks
(a MAI def dominating a non-MAI def, or a src2 use not dominated by any
bridge-copy block), with an early-safe return for parallel candidate
defs.
- hasDstSubregConflict: detects dst subreg writers that cannot be
reclassified to AGPR (non-agnostic writers, or agnostic writers with
non-agnostic orphan uses).
- Exclusion propagates forward along the dst->src2 chain and backward to
MAI reaching-defs of a conflicted src2.
hasUseRequiringVGPR now takes the exclusion set so that excluded MFMAs,
which stay in VGPR form, are correctly treated as VGPR-requiring uses.
initHeuristics is split into a candidate-collection pass and a
heuristics pass over non-excluded candidates.
Add rewrite-mfma-form-safe-guard.mir covering the exclusion branches and
update sched_mfma_rewrite_copies.mir for the new behavior.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 536 ++++++++++++---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 62 +-
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 635 ++++++++++++++++++
.../AMDGPU/sched_mfma_rewrite_copies.mir | 170 +++--
4 files changed, 1232 insertions(+), 171 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4f854beaeebe..e893089412115 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -35,6 +35,7 @@
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/Rematerializer.h"
@@ -1315,48 +1316,108 @@ bool GCNSchedStage::initGCNSchedStage() {
return true;
}
-void RewriteMFMAFormStage::findReachingDefs(
- MachineOperand &UseMO, LiveIntervals *LIS,
- SmallVectorImpl<SlotIndex> &DefIdxs) {
- MachineInstr *UseMI = UseMO.getParent();
- LiveInterval &UseLI = LIS->getInterval(UseMO.getReg());
- VNInfo *VNI = UseLI.getVNInfoAt(LIS->getInstructionIndex(*UseMI));
+static void collectReachingDefsInRange(const LiveRange &QR,
+ MachineBasicBlock *UseMBB,
+ SlotIndex UseIdx,
+ const LiveIntervals *LIS,
+ SmallSet<SlotIndex, 8> &ResultSet) {
+ const VNInfo *VNI = QR.getVNInfoAt(UseIdx);
+ if (!VNI)
+ return;
- // If the def is not a PHI, then it must be the only reaching def.
if (!VNI->isPHIDef()) {
- DefIdxs.push_back(VNI->def);
+ ResultSet.insert(VNI->def);
return;
}
- SmallPtrSet<MachineBasicBlock *, 8> Visited = {UseMI->getParent()};
+ SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<MachineBasicBlock *, 8> Worklist;
-
- // Mark the predecessor blocks for traversal
- for (MachineBasicBlock *PredMBB : UseMI->getParent()->predecessors()) {
- Worklist.push_back(PredMBB);
- Visited.insert(PredMBB);
- }
+ for (MachineBasicBlock *PredMBB : UseMBB->predecessors())
+ if (Visited.insert(PredMBB).second)
+ Worklist.push_back(PredMBB);
while (!Worklist.empty()) {
MachineBasicBlock *CurrMBB = Worklist.pop_back_val();
+ SlotIndex CurrMBBEnd = LIS->getMBBEndIdx(CurrMBB).getPrevSlot();
+ const VNInfo *PredVNI = QR.getVNInfoAt(CurrMBBEnd);
+ if (!PredVNI)
+ continue;
- SlotIndex CurrMBBEnd = LIS->getMBBEndIdx(CurrMBB);
- VNInfo *VNI = UseLI.getVNInfoAt(CurrMBBEnd.getPrevSlot());
-
- MachineBasicBlock *DefMBB = LIS->getMBBFromIndex(VNI->def);
-
- // If there is a def in this block, then add it to the list. This is the
- // reaching def of this path.
- if (!VNI->isPHIDef()) {
- DefIdxs.push_back(VNI->def);
+ if (!PredVNI->isPHIDef()) {
+ // Skip self-referential defs (src2 == dst): the MFMA's own result must
+ // not appear as a reaching def of its own src2 operand.
+ if (SlotIndex::isSameInstr(PredVNI->def, UseIdx))
+ continue;
+ ResultSet.insert(PredVNI->def);
continue;
}
- for (MachineBasicBlock *PredMBB : DefMBB->predecessors()) {
+ MachineBasicBlock *DefMBB = LIS->getMBBFromIndex(PredVNI->def);
+ for (MachineBasicBlock *PredMBB : DefMBB->predecessors())
if (Visited.insert(PredMBB).second)
Worklist.push_back(PredMBB);
+ }
+}
+
+void RewriteMFMAFormStage::findReachingDefs(
+ MachineOperand &UseMO, LiveIntervals *LIS,
+ SmallVectorImpl<SlotIndex> &DefIdxs) {
+ MachineInstr *UseMI = UseMO.getParent();
+ Register UseReg = UseMO.getReg();
+ unsigned UseSubReg = UseMO.getSubReg();
+
+ if (!UseReg.isVirtual() || !LIS->hasInterval(UseReg))
+ return;
+
+ LiveInterval &UseLI = LIS->getInterval(UseReg);
+ SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI);
+
+ // Use a set to deduplicate: a full-reg def appears in every SubRange but
+ // must be inserted into DefIdxs only once.
+ SmallSet<SlotIndex, 8> ResultSet;
+
+ if (UseLI.hasSubRanges()) {
+ const TargetRegisterInfo *TRI = DAG.MRI.getTargetRegisterInfo();
+
+ if (UseSubReg) {
+ // Find the SubRange whose LaneMask fully covers the operand's lanes.
+ // If none does (e.g. register initialised via per-lane subreg writes),
+ // fall back to all overlapping SubRanges to capture every reaching def.
+ LaneBitmask UseLanes = TRI->getSubRegIndexLaneMask(UseSubReg);
+ bool FoundFullCoverage = false;
+ for (LiveInterval::SubRange &SR : UseLI.subranges()) {
+ if ((SR.LaneMask & UseLanes) == UseLanes) {
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ FoundFullCoverage = true;
+ break;
+ }
+ }
+ if (!FoundFullCoverage) {
+ for (LiveInterval::SubRange &SR : UseLI.subranges())
+ if ((SR.LaneMask & UseLanes).any())
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ }
+ } else {
+ // Full-reg use: query every subrange so that partial (subreg) defs on
+ // different lanes are all captured.
+ for (LiveInterval::SubRange &SR : UseLI.subranges())
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ // If the register has SubRanges but none covers the use point,
+ // LiveIntervals is malformed: a full-width def must appear in at least
+ // one SubRange.
+ assert(!ResultSet.empty() &&
+ "hasSubRanges() but no SubRange live at full-reg use: "
+ "LiveInterval construction is inconsistent");
}
+ } else {
+ collectReachingDefsInRange(UseLI, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
}
+
+ DefIdxs.append(ResultSet.begin(), ResultSet.end());
}
void RewriteMFMAFormStage::findReachingUses(
@@ -1365,6 +1426,12 @@ void RewriteMFMAFormStage::findReachingUses(
SlotIndex DefIdx = LIS->getInstructionIndex(*DefMI);
for (MachineOperand &UseMO :
DAG.MRI.use_nodbg_operands(DefMI->getOperand(0).getReg())) {
+ // Skip implicit operands: partial subreg defs carry an implicit use of the
+ // full register for RMW lane preservation, not as a real consumer of the
+ // MFMA dst value. Treating them as reaching uses inserts a spurious bridge
+ // copy before the partial def, corrupting MappedReg's live range.
+ if (UseMO.isImplicit())
+ continue;
SmallVector<SlotIndex, 8> ReachingDefIndexes;
findReachingDefs(UseMO, LIS, ReachingDefIndexes);
@@ -2290,12 +2357,29 @@ void GCNSchedStage::modifyRegionSchedule(unsigned RegionIdx,
DAG.Regions[RegionIdx].first = MIOrder.front();
}
+static unsigned getDefSubReg(const MachineInstr &MI, Register Reg) {
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg() == Reg)
+ return MO.getSubReg();
+ return AMDGPU::NoSubRegister;
+}
+
+/// Returns true if \p MI is a class-agnostic subreg writer (COPY or AV_MOV).
+/// These lower to v_accvgpr_write after AGPR reclassification and are legal.
+static bool isAgnosticSubregWriter(const MachineInstr *MI) {
+ if (MI->isCopy())
+ return true;
+ unsigned Opc = MI->getOpcode();
+ return Opc == AMDGPU::AV_MOV_B32_IMM_PSEUDO ||
+ Opc == AMDGPU::AV_MOV_B64_IMM_PSEUDO;
+}
+
/// Returns true if reaching def \p RD will be in AGPR form after the rewrite
/// and so needs no bridge copy: a candidate MFMA in \p RewriteSet, an
/// AV_MOV_*_IMM_PSEUDO, or a copy from a candidate src2 reg in \p CandSrc2Regs.
/// A non-candidate MFMA stays in VGPR form and still needs a bridge.
static bool isReachingDefAGPRForm(
- MachineInstr *RD, const SmallPtrSetImpl<MachineInstr *> &RewriteSet,
+ MachineInstr *RD, const SmallSetVector<MachineInstr *, 16> &RewriteSet,
const DenseSet<Register> &CandSrc2Regs, const SIInstrInfo &TII) {
if (TII.isMAI(*RD))
return RewriteSet.contains(RD);
@@ -2309,16 +2393,27 @@ static bool isReachingDefAGPRForm(
bool RewriteMFMAFormStage::hasUseRequiringVGPR(
ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallPtrSetImpl<MachineInstr *> &RewriteSet) {
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
for (SlotIndex RDIdx : Src2ReachingDefs) {
const MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ // If RD is a non-excluded MFMA candidate, its dst will be reclassified to
+ // AGPR and Case 2 will insert vreg bridge copies for all non-MAI uses of
+ // its dst. Those uses therefore do not impose a VGPR constraint on src2.
+ // The !ExcludedMFMAs.count(RD) guard is defensive:
+ // propagateExclusionForward ensures that if an excluded MFMA's dst is used
+ // as src2, the consumer MFMA is also excluded and hasUseRequiringVGPR is
+ // never called for it.
+ if (TII->isMAI(*RD) && RewriteSet.contains(RD) && !ExcludedMFMAs.count(RD))
+ continue;
SmallVector<MachineOperand *, 8> ReachingUses;
findReachingUses(RD, DAG.LIS, ReachingUses);
for (const MachineOperand *UseMO : ReachingUses) {
const MachineInstr *UseMI = UseMO->getParent();
if (UseMI->isCopy())
continue;
- if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI))
+ if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI) &&
+ !ExcludedMFMAs.count(UseMI))
continue;
return true;
}
@@ -2354,24 +2449,247 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
return false;
if (AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode()) == -1)
return false;
- // Reject candidates whose users force an unavoidable bridge copy.
- Register DstReg = MI->getOperand(0).getReg();
- for (const MachineOperand &Use : DAG.MRI.use_nodbg_operands(DstReg)) {
- if (!TII->isMAI(*Use.getParent()) && !Use.getParent()->isCopy())
- return false;
- }
return true;
}
+bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
+ Register Src2Reg) const {
+ auto &MDT = DAG.LIS->getDomTree();
+
+ SmallVector<MachineInstr *, 8> MAIMIs;
+ SmallPtrSet<MachineBasicBlock *, 8> BridgeCopyBlocks; // non-MAI def blocks
+
+ for (SlotIndex SI : DefIdxs) {
+ MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
+ if (TII->isMFMA(*MI))
+ MAIMIs.push_back(MI);
+ else
+ BridgeCopyBlocks.insert(MI->getParent());
+ }
+
+ if (BridgeCopyBlocks.empty())
+ return false; // All defs are MAI; no bridge copies needed.
+
+ // Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
+ // The MFMA first writes all lanes (AGPR), then a non-MAI instruction
+ // partially overwrites some lanes. A bridge copy at the non-MAI def
+ // would read an already-AGPR register and must partially update it —
+ // a read-modify-write that the bridge-copy mechanism cannot implement.
+ if (!MAIMIs.empty()) {
+ SmallVector<MachineInstr *, 8> NonMAIMIs;
+ for (SlotIndex SI : DefIdxs) {
+ MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
+ if (!TII->isMFMA(*MI))
+ NonMAIMIs.push_back(MI);
+ }
+
+ for (MachineInstr *M : MAIMIs)
+ for (MachineInstr *N : NonMAIMIs)
+ if (MDT.dominates(M, N))
+ return true;
+
+ // Early-safe return: if every (MAI, non-MAI) pair is parallel and every
+ // MAI def is itself a rewrite candidate, the rewrite is safe without
+ // invoking Check 2.
+ if (!NonMAIMIs.empty()) {
+ bool AllParallelAndCandidates = true;
+ for (MachineInstr *M : MAIMIs) {
+ if (!isRewriteCandidate(M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ for (MachineInstr *N : NonMAIMIs) {
+ if (MDT.dominates(N, M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ }
+ if (!AllParallelAndCandidates)
+ break;
+ }
+ if (AllParallelAndCandidates)
+ return false;
+ }
+ }
+
+ // Check 2: every use of Src2Reg must be dominated by at least one
+ // bridge-copy block; otherwise %MappedReg would be undefined on some path.
+ //
+ // use_nodbg_operands is used here (not findReachingUses) because rewrite()
+ // replaces the src2 operand of the MFMA with %MappedReg uniformly — it does
+ // not distinguish which reaching def flows to which use. %MappedReg is
+ // defined only in bridge-copy blocks (after non-MAI defs); if any use of
+ // Src2Reg is in a block not dominated by any bridge-copy block, %MappedReg
+ // would be undefined on that path regardless of whether the use is reached
+ // by a MAI or non-MAI def. findReachingUses(non-MAI RD) would miss uses
+ // that arrive only via MAI defs or other defs (e.g. IMPLICIT_DEF on a
+ // bypass path), causing a false negative and silent undefined-read.
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
+ bool Covered = any_of(BridgeCopyBlocks, [&](const MachineBasicBlock *B) {
+ return MDT.dominates(B, UseBlock);
+ });
+ if (!Covered)
+ return true;
+ }
+
+ return false;
+}
+
+void RewriteMFMAFormStage::propagateExclusionForward(
+ MachineInstr *Root, SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
+ SmallVector<MachineInstr *, 8> Worklist = {Root};
+ while (!Worklist.empty()) {
+ MachineInstr *ExclMI = Worklist.pop_back_val();
+ MachineOperand &DstMO = ExclMI->getOperand(0);
+ if (!DstMO.isReg() || !DstMO.getReg().isVirtual())
+ continue;
+ Register DstReg = DstMO.getReg();
+ for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(DstReg)) {
+ MachineInstr *UserMI = UseMO.getParent();
+ if (!isRewriteCandidate(UserMI))
+ continue;
+ MachineOperand *UserSrc2 =
+ TII->getNamedOperand(*UserMI, AMDGPU::OpName::src2);
+ if (!UserSrc2 || !UserSrc2->isReg() || UserSrc2->getReg() != DstReg)
+ continue;
+ if (ExcludedMFMAs.insert(UserMI).second) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] exclude downstream MFMA "
+ "(src2 = excluded MFMA dst): "
+ << *UserMI);
+ Worklist.push_back(UserMI);
+ }
+ }
+ }
+}
+
+// rewrite()'s design principle: reclassify DstReg from VGPR to AGPR class so
+// that the MFMA emits its result directly into AGPR, eliminating the need for
+// a post-MFMA VGPR→AGPR copy. For this reclassification to be legal, every
+// def and every use of DstReg throughout its live range must support
+// AGPR-class operands. hasDstSubregConflict checks this before rewriting:
+//
+bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
+ MachineInstr *MFMA) {
+ SmallVector<MachineOperand *, 8> DstReachingUses;
+ findReachingUses(MFMA, DAG.LIS, DstReachingUses);
+ SmallPtrSet<MachineInstr *, 8> CheckedDefs;
+ SmallVector<MachineInstr *, 4> SafeSubregDefs;
+
+ // Phase 1: classify subreg reaching defs.
+ // Non-agnostic subreg def → immediate conflict.
+ // Agnostic (COPY/AV_MOV) subreg def → defer to orphan-use check.
+ for (MachineOperand *RUOp : DstReachingUses) {
+ SmallVector<SlotIndex, 8> ReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, ReachingDefs);
+ for (SlotIndex RDIdx : ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!CheckedDefs.insert(RD).second)
+ continue;
+ if (getDefSubReg(*RD, DstReg) == AMDGPU::NoSubRegister || TII->isMAI(*RD))
+ continue;
+ if (!isAgnosticSubregWriter(RD))
+ return true;
+ SafeSubregDefs.push_back(RD);
+ }
+ }
+
+ // Phase 2: agnostic subreg defs lower to v_accvgpr_write after reclassify,
+ // so their orphan uses read an AGPR sub-register. Only COPY and AV_MOV can
+ // legally source an AGPR sub-register; anything else is a conflict.
+ for (MachineInstr *RD : SafeSubregDefs) {
+ SlotIndex RDIdx = DAG.LIS->getInstructionIndex(*RD);
+ for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(DstReg)) {
+ if (UseMO.isImplicit() || TII->isMAI(*UseMO.getParent()))
+ continue;
+ SmallVector<SlotIndex, 8> UseReachingDefs;
+ findReachingDefs(UseMO, DAG.LIS, UseReachingDefs);
+ if (any_of(UseReachingDefs,
+ [RDIdx](SlotIndex SI) {
+ return SlotIndex::isSameInstr(SI, RDIdx);
+ }) &&
+ !isAgnosticSubregWriter(UseMO.getParent()))
+ return true;
+ }
+ }
+ return false;
+}
+
+SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
+ // Per-MI checks run cheapest-first:
+ // 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
+ // a read-modify-write on AGPR (MAI def dominates non-MAI def), or
+ // absent on some CFG path to a src2 use.
+ // 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
+ // be reclassified to AGPR. Skipped when check 1 already forces
+ // exclusion (!HasConflict &&).
+ // Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
+ // and backward (MAI reaching-defs of a conflicted src2).
+ SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ Register DstReg = MI->getOperand(0).getReg();
+
+ bool HasConflict = false;
+ SmallVector<SlotIndex, 8> Src2Defs;
+ if (Src2->isReg()) {
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ LLVM_DEBUG({
+ dbgs() << "[computeExclusionSet] candidate: " << *MI;
+ dbgs() << " src2 reaching defs (" << Src2Defs.size() << "):\n";
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *D = DAG.LIS->getInstructionFromIndex(SI);
+ dbgs() << " " << SI << " opcode=" << (D ? (int)D->getOpcode() : -1)
+ << "\n";
+ if (D)
+ dbgs() << " " << *D;
+ }
+ });
+ HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg());
+ }
+ bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+
+ if (!HasConflict && !HasDstSubregDef)
+ continue;
+
+ if (ExcludedMFMAs.insert(MI).second) {
+ LLVM_DEBUG(
+ dbgs() << "[computeExclusionSet] exclude MFMA ("
+ << (HasConflict ? "src2 dominance conflict" : "")
+ << (HasConflict && HasDstSubregDef ? " + " : "")
+ << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
+ << "): " << *MI);
+ propagateExclusionForward(MI, ExcludedMFMAs);
+ }
+
+ // Backward: exclude MAI reaching-defs that are themselves candidates.
+ if (HasConflict) {
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
+ if (TII->isMFMA(*DefMI) && isRewriteCandidate(DefMI) &&
+ ExcludedMFMAs.insert(DefMI).second) {
+ LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
+ "(backward from src2 dominance conflict): "
+ << *DefMI);
+ propagateExclusionForward(DefMI, ExcludedMFMAs);
+ }
+ }
+ }
+ }
+ return ExcludedMFMAs;
+}
+
bool RewriteMFMAFormStage::initHeuristics(
std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
bool Changed = false;
- // Collect the candidate group, its members share AGPR-form operands
- // post-rewrite, so reaching defs feeding any member don't need bridge copy.
- SmallPtrSet<MachineInstr *, 16> RewriteSet;
+ // Pass 1: collect candidate group.
+ // RewriteSet/CandSrc2Regs are needed by isReachingDefAGPRForm and
+ // hasUseRequiringVGPR; collect them before any setDesc/setRegClass changes.
+ SmallSetVector<MachineInstr *, 16> RewriteSet;
DenseSet<Register> CandSrc2Regs;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
@@ -2384,86 +2702,98 @@ bool RewriteMFMAFormStage::initHeuristics(
}
}
- // Prepare for the heuristics
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
- continue;
+ // Phase 1: identify candidates that cannot be safely rewritten.
+ SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs =
+ computeExclusionSet(RewriteSet);
- int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
- assert(ReplacementOp != -1);
+ // Pass 2: compute heuristics for non-excluded candidates.
+ for (MachineInstr *MI : RewriteSet) {
+ if (ExcludedMFMAs.count(MI)) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *MI);
+ continue;
+ }
- RewriteCands.push_back({&MI, MI.getOpcode()});
- MI.setDesc(TII->get(ReplacementOp));
+ int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
+ assert(ReplacementOp != -1);
- MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (Src2->isReg()) {
- SmallVector<SlotIndex, 8> Src2ReachingDefs;
- findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
-
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR = hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet);
- Src2NeedsVGPRCache[&MI] = Src2NeedsVGPR;
-
- for (SlotIndex RDIdx : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
- continue;
- CopyForDef.insert(RD);
- }
- }
+ RewriteCands.push_back({MI, MI->getOpcode()});
+ MI->setDesc(TII->get(ReplacementOp));
- MachineOperand &Dst = MI.getOperand(0);
- SmallVector<MachineOperand *, 8> DstReachingUses;
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2->isReg()) {
+ SmallVector<SlotIndex, 8> Src2ReachingDefs;
+ findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
- findReachingUses(&MI, DAG.LIS, DstReachingUses);
+ // If src2 has a use that must remain VGPR, it cannot be reclassified to
+ // AGPR.
+ bool Src2NeedsVGPR =
+ hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet, ExcludedMFMAs);
+ Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
+ for (SlotIndex RDIdx : Src2ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!Src2NeedsVGPR &&
+ isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
continue;
+ // If this reaching def writes into the src2 register itself (possibly
+ // a partial subreg write), the def will be reclassified together with
+ // src2 and does not need a bridge copy.
+ if (any_of(RD->defs(), [&](const MachineOperand &DefOp) {
+ return DefOp.isReg() && DefOp.getReg() == Src2->getReg();
+ }))
+ continue;
+ CopyForDef.insert(RD);
+ }
+ }
+
+ MachineOperand &Dst = MI->getOperand(0);
+ SmallVector<MachineOperand *, 8> DstReachingUses;
- // For any user of the result of the MFMA which is not an MFMA, we
- // insert a copy. For a given register, we will only insert one copy
- // per user block.
- CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
+ findReachingUses(MI, DAG.LIS, DstReachingUses);
- if (TII->isMAI(*UserMI))
- continue;
+ for (MachineOperand *RUOp : DstReachingUses) {
+ MachineInstr *UserMI = RUOp->getParent();
+ // Group members read the AGPR result directly.
+ if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
+ continue;
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
+ // For any user of the result of the MFMA which is not an MFMA, we
+ // insert a copy. For a given register, we will only insert one copy
+ // per user block.
+ CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
+ if (TII->isMAI(*UserMI))
+ continue;
- // For any definition of the user of the MFMA which is not an MFMA,
- // we insert a copy. We do this to transform all the reaching defs
- // of this use to AGPR. By doing this, we can insert a copy from
- // AGPR to VGPR at the user rather than after the MFMA.
- CopyForDef.insert(RD);
- }
- }
+ SmallVector<SlotIndex, 8> DstUsesReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
- // Do the rewrite to allow for updated RP calculation.
- const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
- const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
- DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
- if (Src2->isReg()) {
- // Have to get src types separately since subregs may cause C and D
- // registers to be different types even though the actual operand is
- // the same size.
- const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
- DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ for (SlotIndex RDIndex : DstUsesReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
+ if (TII->isMAI(*RD))
+ continue;
+
+ // For any definition of the user of the MFMA which is not an MFMA,
+ // we insert a copy. We do this to transform all the reaching defs
+ // of this use to AGPR. By doing this, we can insert a copy from
+ // AGPR to VGPR at the user rather than after the MFMA.
+ CopyForDef.insert(RD);
}
- Changed = true;
}
+
+ // Do the rewrite to allow for updated RP calculation.
+ const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
+ const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
+ DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
+ if (Src2->isReg()) {
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ }
+ Changed = true;
}
return Changed;
@@ -2640,7 +2970,7 @@ bool RewriteMFMAFormStage::rewrite(
// Collect the candidate group; its members share AGPR-form operands
// post-rewrite, so reaching defs feeding any member need no bridge copy.
- SmallPtrSet<MachineInstr *, 16> RewriteCandsSet;
+ SmallSetVector<MachineInstr *, 16> RewriteCandsSet;
DenseSet<Register> RewriteSrc2Regs;
for (auto &[MI, OriginalOpcode] : RewriteCands) {
RewriteCandsSet.insert(MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2059f4e6479ff..25a696f3e540b 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -472,6 +472,54 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// \returns true if this MI is a rewrite candidate.
bool isRewriteCandidate(MachineInstr *MI) const;
+ /// Returns true if the src2 reaching defs \p DefIdxs of \p Src2Reg have a
+ /// conflict that prevents safe bridge-copy insertion after non-MAI defs.
+ ///
+ /// rewrite()'s design principle for src2: instead of reclassifying Src2Reg
+ /// to AGPR directly, Case 1 creates a fresh VGPR %MappedReg and inserts a
+ /// bridge COPY (Src2Reg → %MappedReg) after each non-MAI reaching def. The
+ /// MFMA's src2 operand is then replaced with %MappedReg, which rewrite()
+ /// subsequently reclassifies to AGPR. This preserves Src2Reg's VGPR class
+ /// for non-MFMA users while giving the MFMA an AGPR-class src2.
+ ///
+ /// For this strategy to be correct, every bridge COPY must produce a
+ /// well-defined %MappedReg at every src2 use site. Two conditions break
+ /// this requirement:
+ /// Check 1: if a MAI def dominates a non-MAI def, the bridge copy after the
+ /// non-MAI def would need to partially update an already-AGPR register —
+ /// illegal for rewrite()'s COPY model.
+ /// Early-safe return: if all (MAI, non-MAI) pairs are parallel in the CFG
+ /// and every MAI def is itself a candidate, the non-MAI defs can be bridged
+ /// without conflict; Check 2 is skipped.
+ /// Check 2: every use of \p Src2Reg must be dominated by at least one
+ /// non-MAI def block (where a bridge copy is inserted); otherwise %MappedReg
+ /// is undefined on some CFG path to a use.
+ bool hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
+ Register Src2Reg) const;
+
+ /// Returns true if reclassifying \p DstReg to AGPR would require bridge
+ /// copies beyond rewrite()'s capability (full-register copies only).
+ /// Phase 1: non-agnostic subreg writer (e.g. V_MOV) cannot write an AGPR
+ /// sub-register lane → conflict. Agnostic writers (COPY/AV_MOV) lower to
+ /// v_accvgpr_write and are legal but their orphan uses are checked in
+ /// Phase 2. Phase 2: if an orphan use of an agnostic subreg def is
+ /// non-agnostic, it cannot legally source an AGPR sub-register lane →
+ /// conflict.
+ bool hasDstSubregConflict(Register DstReg, MachineInstr *MFMA);
+
+ /// Transitively exclude from \p ExcludedMFMAs any rewrite candidate whose
+ /// src2 is the dst of \p Root or of any already-excluded MFMA.
+ void
+ propagateExclusionForward(MachineInstr *Root,
+ SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
+
+ /// Compute the set of rewrite candidates in \p RewriteSet that must be
+ /// excluded from rewriting due to src2 dominance conflicts or dst subreg
+ /// conflicts. Exclusion propagates forward (dst→src2 chain) and backward
+ /// (MAI reaching-defs of a conflicted src2). No IR is mutated.
+ SmallPtrSet<MachineInstr *, 16>
+ computeExclusionSet(const SmallSetVector<MachineInstr *, 16> &RewriteSet);
+
/// Resets all candidates in \p RewriteCands back to VGPR form.
void resetRewriteCandsToVGPR(
ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
@@ -486,11 +534,15 @@ class RewriteMFMAFormStage : public GCNSchedStage {
void findReachingUses(const MachineInstr *DefMI, LiveIntervals *LIS,
SmallVectorImpl<MachineOperand *> &ReachingUses);
- /// Returns true if the src2 register with reaching defs \p Src2ReachingDefs
- /// has a use other than a group MFMA (in \p RewriteSet) or a copy, which
- /// would keep it in VGPR form rather than let it be reclassified to AGPR.
- bool hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
+ /// Returns true if any reaching def of src2 (\p Src2ReachingDefs) has a use
+ /// that requires the def to stay in VGPR form: any use that is not a COPY and
+ /// not a non-excluded MFMA candidate (in \p RewriteSet but not in \p
+ /// ExcludedMFMAs). Excluded MFMAs are not being rewritten, so their src2
+ /// cannot be reclassified to AGPR.
+ bool
+ hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
public:
bool initGCNSchedStage() override;
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
new file mode 100644
index 0000000000000..58a1ca4c8dd04
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -0,0 +1,635 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: %s -o - | FileCheck %s
+
+--- |
+ define void @test_src2_backedge_bypass_excluded() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_subreg_overwrite_cross_bb() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_avmov_agprform_rewrite() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_buffer_load_early_safe_rewrite() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_selfref_avmov_init() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_dst_subreg_loop_copy_consumer() #0 {
+ entry:
+ br label %loop
+ loop:
+ br i1 undef, label %loop, label %exit
+ exit:
+ ret void
+ }
+ define void @test_dst_subreg_copy_valu_consumer() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_partial_exclude_avmov_src2() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+#
+# Loop with bypass path triggers hasSrc2BridgeConflict.
+#
+# CFG:
+# bb.0 ──► bb.1 (preheader) ──► bb.2 (loop) ──► bb.3 (bridge) ──► (ret)
+# (entry) ↑___________|
+# │ (back-edge)
+# └────────────────────────────────────────────► bb.3 (bypass)
+#
+# %acc reaching defs for MFMA_A src2: {AV_MOV×4 (bb.1, non-MAI), MFMA_C (bb.2, MAI)}.
+# hasSrc2BridgeConflict: bb.1 does not dominate bb.3 (bypass skips bb.1) → excluded.
+# MFMA_C excluded by backward propagation; MFMA_B by forward propagation.
+# All three loop MFMAs stay in vreg form.
+#
+# Without exclusion: %acc reclassified to AGPR; the AV_MOV subreg inits in bb.1
+# have no bridge-copy reader and are DCE'd to dead undef. On the first iteration
+# (bb.0→bb.1→bb.2) MFMA_A reads the IMPLICIT_DEF garbage from bb.0 instead of
+# the intended initial accumulator → silent wrong result on iteration 1.
+#
+# CHECK-LABEL: name: test_src2_backedge_bypass_excluded
+# CHECK: bb.2:
+# CHECK: %r1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: V_MFMA_F32_16X16X4F32_e64
+name: test_src2_backedge_bypass_excluded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs > gfx90a limit 256 → excess pressure.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ ; IMPLICIT_DEF covers %acc on the bypass path (bb.0→bb.3) so the verifier
+ ; is satisfied. bb.1's AV_MOVs kill this def before bb.2, so it does not
+ ; appear in findReachingDefs at MFMA_A and is absent from BridgeCopyBlocks.
+ %acc:vreg_128_align2 = IMPLICIT_DEF
+
+ SCHED_BARRIER 0
+
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ ; Subreg init of %acc (AV_MOV, non-MAI). Creates SubRanges on %acc:
+ ; findReachingDefs returns these as non-MAI reaching defs for MFMA_A src2=%acc.
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+
+ ; MFMA_A: src2=%acc — hasSrc2BridgeConflict fires: bb.1 not dom bb.3 (bypass).
+ %r1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_B: excluded by forward propagation from MFMA_A.
+ %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %r1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_C: redefines %acc (MAI, back-edge def). Excluded by backward propagation.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %r2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ bb.3:
+ ; Explicit use of %acc: hasSrc2BridgeConflict sees this use in bb.3, which is
+ ; not dominated by bb.1 (preheader) due to the bypass path → conflict.
+ %bridge_use:vreg_128_align2 = COPY %acc:vreg_128_align2
+
+ KILL %bridge_use
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+
+---
+#
+# Test: diamond CFG with non-MAI subreg overwrites of MFMA dst blocks rewrite.
+#
+# CFG (diamond):
+#
+# bb.0 ← %acc = MFMA(...)
+# / \
+# bb.1 bb.2 ← bb.1: %acc.sub0 = V_MOV 1 (non-MAI subreg overwrite)
+# \ / bb.2: %acc.sub1 = V_MOV 2 (non-MAI subreg overwrite)
+# bb.3 ← %res = MFMA(..., %acc), %res2 = MFMA(..., %res)
+#
+# %acc dst has non-MAI subreg overwrites (V_MOV in bb.1/bb.2) → excluded.
+# %res/%res2 excluded by propagation (src2 = excluded dst).
+#
+# Primary coverage: computeExclusionSet HasDstSubregDef path — %acc's MFMA has
+# an immediate src2 so hasSrc2BridgeConflict never runs; hasDstSubregConflict is
+# the sole trigger and the root of the entire exclusion chain.
+#
+# Without exclusion: %acc reclassified to AGPR → bridge COPYs inserted in both
+# bb.1 and bb.2 define the same vreg without a PHI; MachineVerifier aborts with
+# "Virtual register defs don't dominate all uses" (dominator violation crash).
+#
+# CHECK-LABEL: name: test_subreg_overwrite_cross_bb
+# CHECK: bb.0:
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: bb.3:
+# CHECK: %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+name: test_subreg_overwrite_cross_bb
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ ; MAI full-reg def of %acc (all 4 lanes) — contributes MAI to ResultSet.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ SCHED_BARRIER 0
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ ; bb.1: non-MAI sub0 overwrite — contributes non-MAI to ResultSet.
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ %acc.sub0:vreg_128_align2 = V_MOV_B32_e32 1, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; bb.2: non-MAI sub1 overwrite — contributes non-MAI to ResultSet.
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc.sub1:vreg_128_align2 = V_MOV_B32_e32 2, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; bb.3: %res/%res2 already excluded by forward propagation from %acc's MFMA.
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; Chain the result into another MFMA (MAI use of %res -> CopyForUse = 0).
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+
+---
+#
+# Diamond CFG where hasSrc2BridgeConflict passes.
+#
+# CFG (diamond):
+#
+# bb.0 ← %acc.subN = AV_MOV_B32_IMM_PSEUDO 0 (per-lane AGPR-form init)
+# / \
+# bb.1 bb.2 ← bb.1: %acc live-through (no def)
+# \ / bb.2: %acc = MFMA_prev(a, b, 0) [MAI]
+# bb.3 ← %res = MFMA_target(a, b, %acc), %res2 = MFMA_chain(a, b, %res)
+#
+# reaching defs for %res src2=%acc: {AV_MOV(bb.0, non-MAI), MFMA_prev(bb.2, MAI)}.
+# hasSrc2BridgeConflict: Check1 NO (bb.2 does not dominate bb.0),
+# Check2 NO (bb.0 dominates bb.3 → bridge covered) → not excluded.
+# AV_MOV and MFMA_prev both AGPR-form → CopyCost=0 → rewrite fires.
+#
+# CHECK-LABEL: name: test_src2_avmov_agprform_rewrite
+# CHECK-NOT: COPY
+# CHECK: bb.2:
+# CHECK: %acc:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: {{.*}}:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_avmov_agprform_rewrite
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %p9:vreg_1024 = IMPLICIT_DEF
+ %p10:vreg_1024 = IMPLICIT_DEF
+ %p11:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %p9, %p10, %p11
+
+ S_ENDPGM 0
+
+---
+#
+# Test: RewriteMFMAFormStage — hasSrc2BridgeConflict early-safe return with diamond CFG.
+#
+# CFG (diamond):
+#
+# bb.0 (entry: pressure fill + branch)
+# / \
+# bb.1 bb.2 ← bb.1: %acc = BUFFER_LOAD (non-MAI)
+# \ / bb.2: %acc = MFMA_prev(a, b, 0) (MAI, src2=imm → candidate)
+# bb.3 ← %res = MFMA_target(a, b, %acc), %res2 = MFMA_chain(a, b, %res)
+#
+# hasSrc2BridgeConflict for MFMA_target: reaching defs = {BUFFER_LOAD, MFMA_prev}.
+# Check 1: MFMA_prev(bb.2) does not dominate BUFFER_LOAD(bb.1) → false.
+# Early-safe return: MFMA_prev is a candidate and BUFFER_LOAD(bb.1) does not
+# dominate MFMA_prev(bb.2) → AllParallelAndCandidates=true → return false;
+# Check 2 not reached. All three MFMAs are rewrite candidates.
+#
+# 9 × vreg_1024 = 288 ArchVGPRs > 256 limit → ExcessArchVGPR fires; rewrite fires.
+# Bridge copy inserted in bb.1 after BUFFER_LOAD.
+# CHECK-LABEL: name: test_src2_buffer_load_early_safe_rewrite
+# CHECK: bb.1:
+# CHECK: %acc:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET
+# CHECK: {{.*}}:areg_128_align2 = COPY %acc
+# CHECK: bb.2:
+# CHECK: {{.*}}:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK-NOT: vgprcd
+name: test_src2_buffer_load_early_safe_rewrite
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs (> gfx90a limit 256); KILL in bb.3 keeps them live.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %rsrc:sgpr_128 = IMPLICIT_DEF
+ %soff:sgpr_32 = IMPLICIT_DEF
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET %rsrc:sgpr_128, %soff:sgpr_32, 0, 0, 0, implicit $exec :: (load (s128))
+
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+
+---
+#
+# Self-referential MFMA (dst == src2) with subreg AV_MOV init in entry block.
+#
+# CFG:
+# bb.0 (entry) ──► bb.1 (loop) ──► bb.2 (exit)
+# ↑___________|
+# (back-edge)
+#
+# src2=%acc reaching defs: {AV_MOV×4 (bb.0, non-MAI), MFMA_A back-edge (MAI)}.
+# hasSrc2BridgeConflict: bb.0 dominates bb.1 → Check 2 covered → false.
+# AV_MOV and MFMA_A both AGPR-form → no bridge copy. Both MFMAs rewritten.
+#
+# CHECK-LABEL: name: test_src2_selfref_avmov_init
+# CHECK-NOT: COPY
+# CHECK: bb.1:
+# CHECK: %acc:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: %r2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_selfref_avmov_init
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0, $sgpr4_sgpr5
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ SCHED_BARRIER 0
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7e000000), %bb.2(0x02000000)
+
+ ; MFMA_A: self-ref (dst=%acc == src2=%acc). Back-edge def is AGPR-form.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_B: src2=%acc is MFMA_A output (AGPR-form) → no bridge copy.
+ %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ KILL %r2
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+
+---
+#
+# Test: MFMA with non-MAI subreg overwrite on dst is excluded from rewrite;
+# sibling MFMA without subreg overwrite is rewritten.
+#
+# %dst1: %dst1.sub0 = V_MOV in same BB after MFMA → excluded (stays vgprcd).
+# Without exclusion: %dst1 reclassified to AGPR; V_MOV still writes vreg sub0,
+# then a bridge COPY reads the whole %dst1:vreg with mixed AGPR/VGPR lanes →
+# wrong result (sub1~sub3 sourced from AGPR, sub0 from VGPR V_MOV partial write).
+# %dst0: no subreg overwrite → rewritten to AGPR form.
+#
+# CHECK-LABEL: name: test_dst_subreg_loop_copy_consumer
+# CHECK: bb.1.loop:
+# CHECK: V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %dst1.sub0:vreg_128_align2 = V_MOV_B32_e32 1
+# CHECK: V_MFMA_F32_16X16X4F32_e64
+name: test_dst_subreg_loop_copy_consumer
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ successors: %bb.1(0x80000000)
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ %f0:vgpr_32 = IMPLICIT_DEF
+ %f1:vgpr_32 = IMPLICIT_DEF
+ %f2:vgpr_32 = IMPLICIT_DEF
+ %f3:vgpr_32 = IMPLICIT_DEF
+ undef %src2.sub0:vreg_128_align2 = COPY %f0:vgpr_32
+ %src2.sub1:vreg_128_align2 = COPY %f1:vgpr_32
+ %src2.sub2:vreg_128_align2 = COPY %f2:vgpr_32
+ %src2.sub3:vreg_128_align2 = COPY %f3:vgpr_32
+
+ S_BRANCH %bb.1
+
+ bb.1.loop:
+ successors: %bb.1(0x78000000), %bb.2(0x08000000)
+
+ SCHED_BARRIER 0
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1.sub0:vreg_128_align2 = V_MOV_B32_e32 1, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ $scc = IMPLICIT_DEF
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.2.exit:
+ %tmp0:vreg_128_align2 = COPY %dst0:vreg_128_align2
+ %tmp1:vreg_128_align2 = COPY %dst1:vreg_128_align2
+ S_ENDPGM 0, implicit %tmp0, implicit %tmp1
+
+---
+#
+# Test: hasDstSubregConflict Phase 2 — MFMA dst subreg written by COPY (agnostic,
+# Phase 1 safe) but read by V_MUL (non-agnostic orphan use) → excluded.
+# propagateExclusionForward excludes downstream chain c0..c3; no rewrite.
+#
+# Without exclusion: %mfma reclassified to AGPR; COPY %mfma.sub2 lowers to
+# v_accvgpr_read, but V_MUL reads the sub-register as VGPR — the bridge COPY
+# inserted after the agnostic subreg def is not jointly dominated by all defs
+# of %mfma → "Use not jointly dominated by defs" LIS crash.
+# CHECK-LABEL: name: test_dst_subreg_copy_valu_consumer
+# CHECK: %mfma:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %mfma.sub2:vreg_128_align2 = COPY %val
+# CHECK: %c0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c3:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+name: test_dst_subreg_copy_valu_consumer
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %val:vgpr_32 = IMPLICIT_DEF
+
+ %mfma:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ %mfma.sub2:vreg_128_align2 = COPY %val:vgpr_32
+
+ %result:vgpr_32 = nofpexcept V_MUL_F32_e32 0, %mfma.sub2:vreg_128_align2, implicit $mode, implicit $exec
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7e000000), %bb.2(0x02000000)
+
+ %c0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %mfma:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c0:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c3:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ KILL %c3, %result
+ S_ENDPGM 0
+
+---
+#
+# Test: RewriteMFMAFormStage — hasUseRequiringVGPR uses pre-exclusion RewriteSet,
+# leading to %acc being directly reclassified to AGPR without a bridge copy for
+# the excluded MFMA_B.
+#
+# Bug trigger path:
+# %acc is initialized via AV_MOV_B32_IMM_PSEUDO subreg writes (AGPR-form).
+# No candidate MFMA writes %acc, so Case 2 (dst-use bridge copies) never
+# processes %acc — the only bridge-copy opportunity is Case 1 (src2 bridge).
+#
+# hasUseRequiringVGPR({AV_MOV×4}, RewriteSet):
+# findReachingUses(AV_MOV.subN) → {MFMA_A, MFMA_B}.
+# MFMA_B: isMAI && in pre-exclusion RewriteSet → "safe" → returns false.
+# → Src2NeedsVGPR=false (WRONG: MFMA_B is excluded and stays VGPR form).
+#
+# isReachingDefAGPRForm(AV_MOV) = true → Src2DefsReplace={}.
+# → %acc directly reclassified to AGPR.
+# → No bridge copy inserted for MFMA_B's src2.
+# → MFMA_B (vgprcd, excluded) reads %acc as AGPR → illegal instruction.
+#
+# MFMA_B is excluded due to HasDstSubregDef (V_MOV subreg write of its dst),
+# NOT HasConflict, so backward propagation does NOT run — no cascade.
+# hasSrc2BridgeConflict for MFMA_B: {AV_MOV×4} all in bb.0 which dominates bb.1
+# → all uses covered → HasConflict=false. HasDstSubregDef=true → excluded.
+#
+# Correct behavior (with fix):
+# hasUseRequiringVGPR treats MFMA_B (excluded) as VGPR-requiring.
+# Src2NeedsVGPR=true → bridge copy for %acc inserted in bb.0.
+# %acc stays VGPR; MFMA_A reads %mapped (AGPR); MFMA_B reads %acc (VGPR).
+#
+# CHECK-LABEL: name: test_partial_exclude_avmov_src2
+# CHECK: bb.0:
+# CHECK: {{.*}}:areg_128_align2 = COPY %acc
+# CHECK: %res_a:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.1:
+# CHECK: %res_b:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca, %srcb, %acc
+name: test_partial_exclude_avmov_src2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; Pressure: 9 × vreg_1024 = 288 ArchVGPRs > gfx90a limit 256 → rewrite fires.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ %acc:vreg_128_align2 = IMPLICIT_DEF
+ %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ SCHED_BARRIER 0
+
+ %res_a:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.1:
+ %res_b:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res_b.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %res_b
+ S_ENDPGM 0, implicit %res_a
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
index 71838881aee99..34cf624961f9e 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
@@ -1794,38 +1794,55 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_ADD_U32_e32_]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = COPY [[DEF17]]
; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = COPY [[DEF18]]
; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = COPY [[DEF19]]
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: $scc = IMPLICIT_DEF
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit killed $scc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
- ; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_ADD_U32_e32_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF17]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF18]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[COPY4]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9]]
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11]]
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6]]
+ ; CHECK-NEXT: KILL [[COPY10]], [[COPY5]], [[COPY12]], [[COPY7]], [[COPY14]], [[COPY9]], [[COPY16]], [[COPY11]], [[COPY6]], [[COPY13]], [[COPY8]], [[COPY15]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
@@ -1836,17 +1853,20 @@ body: |
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.6(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_128_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
; CHECK-NEXT: [[DEF20:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[DEF17]], [[DEF18]], [[DEF19]], [[V_ADD_U32_e32_]], [[V_ADD_U32_e32_3]]
+ ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[COPY17]], [[COPY19]], [[COPY18]], [[V_ADD_U32_e32_1]], [[V_ADD_U32_e32_3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -2470,6 +2490,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -2481,27 +2502,34 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: KILL [[COPY2]], [[COPY5]], [[COPY3]], [[COPY6]], [[COPY4]], [[COPY1]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[DS_READ_B128_gfx9_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[COPY7]], 0, 0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[DS_READ_B128_gfx9_]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY7]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -2592,6 +2620,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -2603,27 +2632,34 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: KILL [[COPY2]], [[COPY5]], [[COPY3]], [[COPY6]], [[COPY4]], [[COPY1]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[DS_READ_B128_gfx9_]].sub0, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[COPY7]].sub0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[DS_READ_B128_gfx9_]], [[V_ADD_U32_e32_]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY7]], [[V_ADD_U32_e32_]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -3561,6 +3597,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -3570,10 +3607,10 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000)
@@ -3584,21 +3621,27 @@ body: |
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 256, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub0, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub1, 256, 0, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 256, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub1, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub0, 256, 0, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[DS_READ_B128_gfx9_]], 0, 0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY3]], [[COPY5]], [[COPY4]], [[COPY6]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -5474,23 +5517,24 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DEF16]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF16]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]]
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]].sub0, [[DEF15]], implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[COPY1]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY2]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
>From 68fac5b253a27ac5559cb90763f960ae05ff2eb9 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Wed, 8 Jul 2026 15:51:11 +0000
Subject: [PATCH 2/5] [AMDGPU] Improve src2 bridge-conflict check with joint
bridge coverage
Rework hasSrc2BridgeConflict Check 2 to test path coverage via a backward
CFG walk over the union of non-MAI reaching-def blocks (bridge blocks)
across all candidates sharing the same src2, instead of requiring a single
dominating bridge block. computeExclusionSet precomputes this per-src2
bridge-block union and passes it in.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 133 ++++++++++++--------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 15 ++-
2 files changed, 88 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index e893089412115..b2c97dedbceac 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2452,23 +2452,23 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
return true;
}
-bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
- Register Src2Reg) const {
+bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
+ ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const {
auto &MDT = DAG.LIS->getDomTree();
SmallVector<MachineInstr *, 8> MAIMIs;
- SmallPtrSet<MachineBasicBlock *, 8> BridgeCopyBlocks; // non-MAI def blocks
-
+ SmallVector<MachineInstr *, 8> NonMAIMIs;
for (SlotIndex SI : DefIdxs) {
MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
if (TII->isMFMA(*MI))
MAIMIs.push_back(MI);
else
- BridgeCopyBlocks.insert(MI->getParent());
+ NonMAIMIs.push_back(MI);
}
- if (BridgeCopyBlocks.empty())
- return false; // All defs are MAI; no bridge copies needed.
+ if (NonMAIMIs.empty())
+ return false; // All reaching defs are MAI; no bridge copies needed.
// Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
// The MFMA first writes all lanes (AGPR), then a non-MAI instruction
@@ -2476,13 +2476,6 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
// would read an already-AGPR register and must partially update it —
// a read-modify-write that the bridge-copy mechanism cannot implement.
if (!MAIMIs.empty()) {
- SmallVector<MachineInstr *, 8> NonMAIMIs;
- for (SlotIndex SI : DefIdxs) {
- MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
- if (!TII->isMFMA(*MI))
- NonMAIMIs.push_back(MI);
- }
-
for (MachineInstr *M : MAIMIs)
for (MachineInstr *N : NonMAIMIs)
if (MDT.dominates(M, N))
@@ -2491,45 +2484,60 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
// Early-safe return: if every (MAI, non-MAI) pair is parallel and every
// MAI def is itself a rewrite candidate, the rewrite is safe without
// invoking Check 2.
- if (!NonMAIMIs.empty()) {
- bool AllParallelAndCandidates = true;
- for (MachineInstr *M : MAIMIs) {
- if (!isRewriteCandidate(M)) {
+ bool AllParallelAndCandidates = true;
+ for (MachineInstr *M : MAIMIs) {
+ if (!isRewriteCandidate(M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ for (MachineInstr *N : NonMAIMIs) {
+ if (MDT.dominates(N, M)) {
AllParallelAndCandidates = false;
break;
}
- for (MachineInstr *N : NonMAIMIs) {
- if (MDT.dominates(N, M)) {
- AllParallelAndCandidates = false;
- break;
- }
- }
- if (!AllParallelAndCandidates)
- break;
}
- if (AllParallelAndCandidates)
- return false;
+ if (!AllParallelAndCandidates)
+ break;
}
+ if (AllParallelAndCandidates)
+ return false;
}
- // Check 2: every use of Src2Reg must be dominated by at least one
- // bridge-copy block; otherwise %MappedReg would be undefined on some path.
+ // Check 2: every entry->use path of Src2Reg must cross a bridge block (where
+ // rewrite() inserts the bridge copy); otherwise %MappedReg is undefined on
+ // the bypassing path. BridgeBlocks (precomputed in computeExclusionSet) is
+ // the union of non-MAI *reaching-def* blocks of Src2Reg across all candidates
+ // sharing it -- see computeExclusionSet for why the whole-register union and
+ // reaching-def (not all-def) blocks are both required.
//
- // use_nodbg_operands is used here (not findReachingUses) because rewrite()
- // replaces the src2 operand of the MFMA with %MappedReg uniformly — it does
- // not distinguish which reaching def flows to which use. %MappedReg is
- // defined only in bridge-copy blocks (after non-MAI defs); if any use of
- // Src2Reg is in a block not dominated by any bridge-copy block, %MappedReg
- // would be undefined on that path regardless of whether the use is reached
- // by a MAI or non-MAI def. findReachingUses(non-MAI RD) would miss uses
- // that arrive only via MAI defs or other defs (e.g. IMPLICIT_DEF on a
- // bypass path), causing a false negative and silent undefined-read.
+ // Bridge blocks must *jointly* cover every path: two blocks may each cover a
+ // disjoint set of paths with neither dominating the use, so a plain dominance
+ // test is too strict. A backward walk from the use block that stops at
+ // bridge blocks decides this -- reaching entry means an uncovered path
+ // exists. All uses (use_nodbg_operands, not just src2 consumers) must be
+ // covered: a plain COPY of Src2Reg also reads %MappedReg after rewrite.
+ auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
+ if (BridgeBlocks.contains(UseBlock))
+ return true;
+ SmallPtrSet<const MachineBasicBlock *, 16> Visited;
+ SmallVector<const MachineBasicBlock *, 16> Worklist(UseBlock->pred_begin(),
+ UseBlock->pred_end());
+ while (!Worklist.empty()) {
+ const MachineBasicBlock *B = Worklist.pop_back_val();
+ if (BridgeBlocks.contains(B))
+ continue; // This path is covered; do not walk past the bridge block.
+ if (B->pred_empty())
+ return false; // Reached entry without crossing a bridge block.
+ if (!Visited.insert(B).second)
+ continue;
+ Worklist.append(B->pred_begin(), B->pred_end());
+ }
+ return true;
+ };
+
for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
- bool Covered = any_of(BridgeCopyBlocks, [&](const MachineBasicBlock *B) {
- return MDT.dominates(B, UseBlock);
- });
- if (!Covered)
+ if (!CoveredByBridgeSet(UseBlock))
return true;
}
@@ -2627,6 +2635,31 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
// Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
// and backward (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
+
+ // Precompute, per shared src2 register, the union of its non-MAI reaching-def
+ // blocks across all candidates -- exactly where rewrite() inserts bridge
+ // copies. Check 2 needs the whole-register union, not one candidate's
+ // reaching defs, since a sibling candidate's def in another block also
+ // supplies a value to a shared use. Reaching-def (not all-def) blocks are
+ // used so defs killed on a bypass path and entry IMPLICIT_DEFs stay excluded,
+ // keeping a genuine bypass reported as a conflict.
+ DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
+ Src2BridgeBlocks;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2 || !Src2->isReg())
+ continue;
+ Register Src2Reg = Src2->getReg();
+ auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ SmallVector<SlotIndex, 8> Src2Defs;
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
+ if (RD && !TII->isMFMA(*RD))
+ Blocks.insert(RD->getParent());
+ }
+ }
+
for (MachineInstr *MI : RewriteSet) {
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
Register DstReg = MI->getOperand(0).getReg();
@@ -2635,18 +2668,8 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
SmallVector<SlotIndex, 8> Src2Defs;
if (Src2->isReg()) {
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- LLVM_DEBUG({
- dbgs() << "[computeExclusionSet] candidate: " << *MI;
- dbgs() << " src2 reaching defs (" << Src2Defs.size() << "):\n";
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *D = DAG.LIS->getInstructionFromIndex(SI);
- dbgs() << " " << SI << " opcode=" << (D ? (int)D->getOpcode() : -1)
- << "\n";
- if (D)
- dbgs() << " " << *D;
- }
- });
- HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg());
+ HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg(),
+ Src2BridgeBlocks[Src2->getReg()]);
}
bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 25a696f3e540b..900b50c9a716d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -491,11 +491,16 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Early-safe return: if all (MAI, non-MAI) pairs are parallel in the CFG
/// and every MAI def is itself a candidate, the non-MAI defs can be bridged
/// without conflict; Check 2 is skipped.
- /// Check 2: every use of \p Src2Reg must be dominated by at least one
- /// non-MAI def block (where a bridge copy is inserted); otherwise %MappedReg
- /// is undefined on some CFG path to a use.
- bool hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
- Register Src2Reg) const;
+ /// Check 2: on every path from the entry to a use of \p Src2Reg, some bridge
+ /// block in \p BridgeBlocks (a non-MAI def block of Src2Reg, where a bridge
+ /// copy is inserted) must be crossed; otherwise %MappedReg is undefined on
+ /// that path. \p BridgeBlocks is precomputed in computeExclusionSet as the
+ /// union, over all candidates sharing Src2Reg, of the non-MAI reaching-def
+ /// blocks of their src2 — this is what rewrite() actually inserts copies
+ /// after, so a single candidate's own reaching defs are not enough.
+ bool hasSrc2BridgeConflict(
+ ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const;
/// Returns true if reclassifying \p DstReg to AGPR would require bridge
/// copies beyond rewrite()'s capability (full-register copies only).
>From d691e11217b467dd2fa6828c8f011c9d7e8712f6 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 04:28:53 +0000
Subject: [PATCH 3/5] [AMDGPU] Add src2 connectivity check to MFMA form rewrite
guard
Extend hasSrc2BridgeConflict with a connectivity check: if src2 has >=2
mutually non-dominating non-AGPR-form partial-subreg reaching-def blocks,
redirecting uses to %MappedReg would fragment the original live interval.
Runs independently of the early-safe path.
Add regression tests test_src2_shared_diamond_joint_coverage and
test_earlysafe_connectivity_fragment.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 110 +++++++----
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 8 +-
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 187 ++++++++++++++++++
3 files changed, 270 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b2c97dedbceac..8f30ab515cdc3 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2454,7 +2454,9 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
- const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const {
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
+ const {
auto &MDT = DAG.LIS->getDomTree();
SmallVector<MachineInstr *, 8> MAIMIs;
@@ -2475,15 +2477,19 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
// partially overwrites some lanes. A bridge copy at the non-MAI def
// would read an already-AGPR register and must partially update it —
// a read-modify-write that the bridge-copy mechanism cannot implement.
+ bool SkipCheck2 = false;
if (!MAIMIs.empty()) {
for (MachineInstr *M : MAIMIs)
for (MachineInstr *N : NonMAIMIs)
if (MDT.dominates(M, N))
return true;
- // Early-safe return: if every (MAI, non-MAI) pair is parallel and every
- // MAI def is itself a rewrite candidate, the rewrite is safe without
- // invoking Check 2.
+ // Early-safe: if every MAI reaching def is a candidate and parallel to
+ // every non-MAI def (Check 1 ruled out MAI dominating non-MAI; here the
+ // reverse), then %MappedReg is defined on every path to a use -- a
+ // candidate MAI gives an AGPR value directly, a non-MAI def carries a
+ // bridge copy -- so Check 2 can be skipped. Check 3 is independent and
+ // must still run.
bool AllParallelAndCandidates = true;
for (MachineInstr *M : MAIMIs) {
if (!isRewriteCandidate(M)) {
@@ -2500,22 +2506,17 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
break;
}
if (AllParallelAndCandidates)
- return false;
+ SkipCheck2 = true;
}
- // Check 2: every entry->use path of Src2Reg must cross a bridge block (where
- // rewrite() inserts the bridge copy); otherwise %MappedReg is undefined on
- // the bypassing path. BridgeBlocks (precomputed in computeExclusionSet) is
- // the union of non-MAI *reaching-def* blocks of Src2Reg across all candidates
- // sharing it -- see computeExclusionSet for why the whole-register union and
- // reaching-def (not all-def) blocks are both required.
- //
- // Bridge blocks must *jointly* cover every path: two blocks may each cover a
- // disjoint set of paths with neither dominating the use, so a plain dominance
- // test is too strict. A backward walk from the use block that stops at
- // bridge blocks decides this -- reaching entry means an uncovered path
- // exists. All uses (use_nodbg_operands, not just src2 consumers) must be
- // covered: a plain COPY of Src2Reg also reads %MappedReg after rewrite.
+ // Check 2: every entry->use path of Src2Reg must hit a bridge block (a
+ // non-MAI def block where rewrite() inserts the bridge copy; the use block
+ // itself counts), else %MappedReg is undefined on the bypassing path. Bridge
+ // blocks must *jointly* cover every path -- one may not dominate the use, so
+ // a dominance test is too strict; a backward walk stopping at bridge blocks
+ // that reaches entry means an uncovered path exists. Every use counts (a
+ // plain COPY of Src2Reg also reads %MappedReg). BridgeBlocks: see
+ // computeExclusionSet.
auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
if (BridgeBlocks.contains(UseBlock))
return true;
@@ -2535,9 +2536,36 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
return true;
};
- for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
- const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
- if (!CoveredByBridgeSet(UseBlock))
+ if (!SkipCheck2) {
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
+ if (!CoveredByBridgeSet(UseBlock))
+ return true;
+ }
+ }
+
+ // Check 3: original-register connectivity. rewrite() redirects Src2Reg's
+ // uses to %MappedReg, so if non-AGPR-form partial subreg defs sit in >=2
+ // blocks with none dominating all uses, Src2Reg's live interval splits into
+ // disconnected fragments. AGPR-form defs are excluded from
+ // RedefPartialBlocks (skipped by Case1), preserving the diamond
+ // joint-coverage case.
+ if (RedefPartialBlocks.size() >= 2) {
+ bool SingleDominator = false;
+ for (const MachineBasicBlock *B : RedefPartialBlocks) {
+ bool DomAllUses = true;
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ if (!MDT.dominates(B, UseMO.getParent()->getParent())) {
+ DomAllUses = false;
+ break;
+ }
+ }
+ if (DomAllUses) {
+ SingleDominator = true;
+ break;
+ }
+ }
+ if (!SingleDominator)
return true;
}
@@ -2627,8 +2655,9 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
// Per-MI checks run cheapest-first:
// 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
- // a read-modify-write on AGPR (MAI def dominates non-MAI def), or
- // absent on some CFG path to a src2 use.
+ // a read-modify-write on AGPR (MAI def dominates non-MAI def), the
+ // bridge value is absent on some CFG path to a src2 use, or the
+ // redirected original register splits into disconnected fragments.
// 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
// be reclassified to AGPR. Skipped when check 1 already forces
// exclusion (!HasConflict &&).
@@ -2636,27 +2665,39 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
// and backward (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
- // Precompute, per shared src2 register, the union of its non-MAI reaching-def
- // blocks across all candidates -- exactly where rewrite() inserts bridge
- // copies. Check 2 needs the whole-register union, not one candidate's
- // reaching defs, since a sibling candidate's def in another block also
- // supplies a value to a shared use. Reaching-def (not all-def) blocks are
- // used so defs killed on a bypass path and entry IMPLICIT_DEFs stay excluded,
- // keeping a genuine bypass reported as a conflict.
+ // Per src2 register, union reaching-def blocks across all candidates (one
+ // candidate sees only one path): non-MFMA -> Src2BridgeBlocks (Check 2), plus
+ // non-AGPR-form partial-subreg -> Src2RedefPartialBlocks (Check 3). Reaching-
+ // def (not all-def) keeps bypass-killed and entry IMPLICIT_DEFs out.
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2BridgeBlocks;
+ DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
+ Src2RedefPartialBlocks;
+ DenseSet<Register> CandSrc2Regs;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
for (MachineInstr *MI : RewriteSet) {
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
if (!Src2 || !Src2->isReg())
continue;
Register Src2Reg = Src2->getReg();
auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
SmallVector<SlotIndex, 8> Src2Defs;
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
for (SlotIndex SI : Src2Defs) {
MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
- if (RD && !TII->isMFMA(*RD))
- Blocks.insert(RD->getParent());
+ if (!RD || TII->isMFMA(*RD))
+ continue;
+ Blocks.insert(RD->getParent());
+ // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
+ // (AGPR-form skipped by Case1; full-width defs stay connected).
+ if (!isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII) &&
+ getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
+ RedefBlocks.insert(RD->getParent());
}
}
@@ -2668,8 +2709,9 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
SmallVector<SlotIndex, 8> Src2Defs;
if (Src2->isReg()) {
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg(),
- Src2BridgeBlocks[Src2->getReg()]);
+ HasConflict = hasSrc2BridgeConflict(
+ Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
+ Src2RedefPartialBlocks[Src2->getReg()]);
}
bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 900b50c9a716d..5909ba4d28c8c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -498,9 +498,15 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// union, over all candidates sharing Src2Reg, of the non-MAI reaching-def
/// blocks of their src2 — this is what rewrite() actually inserts copies
/// after, so a single candidate's own reaching defs are not enough.
+ /// Check 3: if \p RedefPartialBlocks (non-AGPR-form partial subreg
+ /// reaching-def blocks of Src2Reg) holds >=2 mutually non-dominating blocks
+ /// with no single block dominating all uses, redirecting uses to %MappedReg
+ /// splits the original Src2Reg live interval into disconnected components.
bool hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
- const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const;
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
+ const;
/// Returns true if reclassifying \p DstReg to AGPR would require bridge
/// copies beyond rewrite()'s capability (full-register copies only).
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
index 58a1ca4c8dd04..2d81741ec9759 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -40,6 +40,14 @@
entry:
unreachable
}
+ define void @test_src2_shared_diamond_joint_coverage() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_earlysafe_connectivity_fragment() #0 {
+ entry:
+ unreachable
+ }
attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
...
@@ -633,3 +641,182 @@ body: |
KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %res_b
S_ENDPGM 0, implicit %res_a
+...
+
+---
+#
+# Shared-src2 diamond where the bridge blocks JOINTLY cover the merge use, but
+# no single block dominates it. This is the opt_bug shape and it exercises the
+# whole-register bridge-set union in computeExclusionSet / hasSrc2BridgeConflict.
+#
+# CFG (diamond):
+#
+# bb.0
+# / \
+# bb.1 bb.2 ← bb.1: %acc.subN = AV_MOV 0 (non-MAI init, left branch)
+# \ / bb.2: %acc.subN = AV_MOV 0 (non-MAI init, right branch)
+# bb.3 ← %res = MFMA(a, b, %acc) (src2 = %acc)
+# %res2 = MFMA(a, b, %acc) (src2 = %acc, second consumer)
+#
+# %acc has two non-MAI def blocks: bb.1 (left) and bb.2 (right). Neither
+# dominates bb.3, but together they cover every entry->bb.3 path. The old
+# per-candidate check built BridgeCopyBlocks from a single candidate's reaching
+# defs and wrongly reported "uncovered". The union across both candidates that
+# share %acc as src2 covers bb.3 → no conflict → both MFMAs rewrite to AGPR.
+#
+# CHECK-LABEL: name: test_src2_shared_diamond_joint_coverage
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: %res2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK-NOT: vgprcd
+name: test_src2_shared_diamond_joint_coverage
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 x vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ ; Left branch: full-width non-MAI init of %acc.
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; Right branch: full-width non-MAI init of %acc.
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; Merge: two MFMAs share %acc as src2. Each is reached by only one branch's
+ ; init in its own reaching-def set; only the cross-candidate union covers bb.3.
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+...
+
+---
+#
+# The early-safe path (skipping the bridge dominance/coverage checks) must NOT
+# suppress the connectivity check. bb.5's MAI def is parallel to every non-MAI
+# def, so hasSrc2BridgeConflict takes the early-safe path and skips bridge
+# coverage -- but connectivity still runs: the two agnostic partial-subreg COPY
+# defs in bb.2/bb.3, neither dominating the bb.6 use, fragment %acc's live range
+# -> conflict -> both MFMAs stay in vgprcd form.
+#
+# CFG:
+#
+# bb.0
+# / \
+# bb.1 bb.5 ← bb.5: %acc = MFMA (MAI full-reg def, candidate)
+# / \ \
+# bb.2 bb.3 \ ← bb.2/bb.3: %acc.subN = COPY (agnostic partial defs)
+# \ / \
+# bb.4 \
+# \ /
+# \ /
+# bb.6 ← %res = MFMA(a, b, %acc) (src2 = %acc)
+#
+# CHECK-LABEL: name: test_earlysafe_connectivity_fragment
+# CHECK: bb.5:
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: bb.6:
+# CHECK: %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: V_MFMA_F32_16X16X4F32_e64
+name: test_earlysafe_connectivity_fragment
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 x vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %accinit:vreg_128_align2 = IMPLICIT_DEF
+ %cpsrc0:vgpr_32 = IMPLICIT_DEF
+ %cpsrc1:vgpr_32 = IMPLICIT_DEF
+ %cpsrc2:vgpr_32 = IMPLICIT_DEF
+ %cpsrc3:vgpr_32 = IMPLICIT_DEF
+
+ S_CBRANCH_SCC1 %bb.5, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ ; branch A: agnostic partial-subreg def (fragments live range, not a subreg-def conflict).
+ bb.2:
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
+
+ ; branch B: agnostic partial-subreg def (fragments live range).
+ bb.3:
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.6(0x80000000)
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %accinit:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.6
+
+ bb.6:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
>From 5461be0d51205b2de007a02436f1afaec14f5f87 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 07:11:19 +0000
Subject: [PATCH 4/5] [AMDGPU] Iterate MFMA-form exclusion to a fixpoint
computeExclusionSet classified src2 reaching defs in a single pass.
Excluding one candidate makes its dst/src2 no longer AGPR-form, which can
expose a conflict in another candidate the single pass never revisits --
letting an unsafe rewrite through and splitting a live interval. Instead,
recompute the block maps from the still-live candidates and iterate until
a round adds nothing; exclusions only grow over a finite set, so it
terminates.
Add test_src2_exclusion_fixpoint covering the cascade where A forces B.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 199 ++++++++++--------
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 97 +++++++++
2 files changed, 206 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 8f30ab515cdc3..df44f4c525bf3 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2472,11 +2472,9 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
if (NonMAIMIs.empty())
return false; // All reaching defs are MAI; no bridge copies needed.
- // Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
- // The MFMA first writes all lanes (AGPR), then a non-MAI instruction
- // partially overwrites some lanes. A bridge copy at the non-MAI def
- // would read an already-AGPR register and must partially update it —
- // a read-modify-write that the bridge-copy mechanism cannot implement.
+ // Check 1: MAI def dominates non-MAI def. The MFMA writes all lanes (AGPR),
+ // then a non-MAI def partially overwrites some; a bridge copy there would be
+ // a read-modify-write on AGPR, which the bridge mechanism cannot implement.
bool SkipCheck2 = false;
if (!MAIMIs.empty()) {
for (MachineInstr *M : MAIMIs)
@@ -2510,12 +2508,11 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
}
// Check 2: every entry->use path of Src2Reg must hit a bridge block (a
- // non-MAI def block where rewrite() inserts the bridge copy; the use block
- // itself counts), else %MappedReg is undefined on the bypassing path. Bridge
- // blocks must *jointly* cover every path -- one may not dominate the use, so
- // a dominance test is too strict; a backward walk stopping at bridge blocks
- // that reaches entry means an uncovered path exists. Every use counts (a
- // plain COPY of Src2Reg also reads %MappedReg). BridgeBlocks: see
+ // non-MAI def block where rewrite() inserts the bridge copy, or the use block
+ // itself), else %MappedReg is undefined on the bypassing path. Bridge blocks
+ // must *jointly* cover every path (dominance is too strict), so a backward
+ // walk that reaches entry without crossing one proves an uncovered path.
+ // Every use counts, incl. a plain COPY. BridgeBlocks: see
// computeExclusionSet.
auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
if (BridgeBlocks.contains(UseBlock))
@@ -2545,11 +2542,10 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
}
// Check 3: original-register connectivity. rewrite() redirects Src2Reg's
- // uses to %MappedReg, so if non-AGPR-form partial subreg defs sit in >=2
- // blocks with none dominating all uses, Src2Reg's live interval splits into
- // disconnected fragments. AGPR-form defs are excluded from
- // RedefPartialBlocks (skipped by Case1), preserving the diamond
- // joint-coverage case.
+ // uses to %MappedReg, so non-AGPR-form partial subreg defs in >=2 blocks with
+ // none dominating all uses split Src2Reg's live interval into disconnected
+ // fragments. AGPR-form defs are absent from RedefPartialBlocks (skipped by
+ // Case1), preserving the diamond joint-coverage case.
if (RedefPartialBlocks.size() >= 2) {
bool SingleDominator = false;
for (const MachineBasicBlock *B : RedefPartialBlocks) {
@@ -2599,12 +2595,9 @@ void RewriteMFMAFormStage::propagateExclusionForward(
}
}
-// rewrite()'s design principle: reclassify DstReg from VGPR to AGPR class so
-// that the MFMA emits its result directly into AGPR, eliminating the need for
-// a post-MFMA VGPR→AGPR copy. For this reclassification to be legal, every
-// def and every use of DstReg throughout its live range must support
-// AGPR-class operands. hasDstSubregConflict checks this before rewriting:
-//
+// rewrite() reclassifies DstReg to AGPR so the MFMA writes its result directly
+// into AGPR (no post-MFMA copy). That is legal only if every def and use of
+// DstReg across its live range supports AGPR-class operands; check that here.
bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
MachineInstr *MFMA) {
SmallVector<MachineOperand *, 8> DstReachingUses;
@@ -2653,16 +2646,10 @@ bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
- // Per-MI checks run cheapest-first:
- // 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
- // a read-modify-write on AGPR (MAI def dominates non-MAI def), the
- // bridge value is absent on some CFG path to a src2 use, or the
- // redirected original register splits into disconnected fragments.
- // 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
- // be reclassified to AGPR. Skipped when check 1 already forces
- // exclusion (!HasConflict &&).
- // Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
- // and backward (MAI reaching-defs of a conflicted src2).
+ // Per-MI checks, cheapest-first: (1) hasSrc2BridgeConflict, then
+ // (2) hasDstSubregConflict (skipped when 1 already excludes). Exclusion
+ // propagates forward (dst->src2 via propagateExclusionForward) and backward
+ // (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
// Per src2 register, union reaching-def blocks across all candidates (one
@@ -2673,75 +2660,107 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
Src2BridgeBlocks;
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2RedefPartialBlocks;
- DenseSet<Register> CandSrc2Regs;
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (!Src2 || !Src2->isReg())
- continue;
- Register Src2Reg = Src2->getReg();
- auto &Blocks = Src2BridgeBlocks[Src2Reg];
- auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
- SmallVector<SlotIndex, 8> Src2Defs;
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
- if (!RD || TII->isMFMA(*RD))
+
+ // Rebuild the block maps from the still-live candidates \p Active: excluding
+ // a candidate makes its src2/MAI def non-AGPR-form (isReachingDefAGPRForm
+ // reads Active), which can add new Check 3 redef sources -- so the full
+ // pre-exclusion set would miss them.
+ auto recomputeBlocks = [&](const SmallSetVector<MachineInstr *, 16> &Active) {
+ Src2BridgeBlocks.clear();
+ Src2RedefPartialBlocks.clear();
+ DenseSet<Register> CandSrc2Regs;
+ for (MachineInstr *MI : Active) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
+ for (MachineInstr *MI : Active) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2 || !Src2->isReg())
continue;
- Blocks.insert(RD->getParent());
- // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
- // (AGPR-form skipped by Case1; full-width defs stay connected).
- if (!isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII) &&
- getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
- RedefBlocks.insert(RD->getParent());
+ Register Src2Reg = Src2->getReg();
+ auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
+ SmallVector<SlotIndex, 8> Src2Defs;
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
+ if (!RD || TII->isMFMA(*RD))
+ continue;
+ Blocks.insert(RD->getParent());
+ // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
+ // (AGPR-form skipped by Case1; full-width defs stay connected).
+ if (!isReachingDefAGPRForm(RD, Active, CandSrc2Regs, *TII) &&
+ getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
+ RedefBlocks.insert(RD->getParent());
+ }
}
- }
+ };
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- Register DstReg = MI->getOperand(0).getReg();
+ // Fixpoint: each exclusion can expose new conflicts in others (see
+ // recomputeBlocks). Iterate to convergence; exclusions only grow over a
+ // finite candidate set, so this terminates.
+ bool Changed;
+ do {
+ Changed = false;
- bool HasConflict = false;
- SmallVector<SlotIndex, 8> Src2Defs;
- if (Src2->isReg()) {
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- HasConflict = hasSrc2BridgeConflict(
- Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
- Src2RedefPartialBlocks[Src2->getReg()]);
- }
- bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+ SmallSetVector<MachineInstr *, 16> Active;
+ for (MachineInstr *MI : RewriteSet)
+ if (!ExcludedMFMAs.count(MI))
+ Active.insert(MI);
- if (!HasConflict && !HasDstSubregDef)
- continue;
+ recomputeBlocks(Active);
- if (ExcludedMFMAs.insert(MI).second) {
- LLVM_DEBUG(
- dbgs() << "[computeExclusionSet] exclude MFMA ("
- << (HasConflict ? "src2 dominance conflict" : "")
- << (HasConflict && HasDstSubregDef ? " + " : "")
- << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
- << "): " << *MI);
- propagateExclusionForward(MI, ExcludedMFMAs);
- }
+ for (MachineInstr *MI : Active) {
+ // May have been excluded by forward/backward propagation earlier this
+ // round.
+ if (ExcludedMFMAs.count(MI))
+ continue;
+
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ Register DstReg = MI->getOperand(0).getReg();
+
+ bool HasConflict = false;
+ SmallVector<SlotIndex, 8> Src2Defs;
+ if (Src2->isReg()) {
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ HasConflict = hasSrc2BridgeConflict(
+ Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
+ Src2RedefPartialBlocks[Src2->getReg()]);
+ }
+ bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+
+ if (!HasConflict && !HasDstSubregDef)
+ continue;
+
+ if (ExcludedMFMAs.insert(MI).second) {
+ LLVM_DEBUG(
+ dbgs() << "[computeExclusionSet] exclude MFMA ("
+ << (HasConflict ? "src2 dominance conflict" : "")
+ << (HasConflict && HasDstSubregDef ? " + " : "")
+ << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
+ << "): " << *MI);
+ propagateExclusionForward(MI, ExcludedMFMAs);
+ Changed = true;
+ }
- // Backward: exclude MAI reaching-defs that are themselves candidates.
- if (HasConflict) {
+ // Backward: exclude MAI reaching-defs that are themselves candidates.
+ if (!HasConflict)
+ continue;
for (SlotIndex SI : Src2Defs) {
MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
- if (TII->isMFMA(*DefMI) && isRewriteCandidate(DefMI) &&
- ExcludedMFMAs.insert(DefMI).second) {
- LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
- "(backward from src2 dominance conflict): "
- << *DefMI);
- propagateExclusionForward(DefMI, ExcludedMFMAs);
- }
+ if (!TII->isMFMA(*DefMI) || !isRewriteCandidate(DefMI) ||
+ !ExcludedMFMAs.insert(DefMI).second)
+ continue;
+ LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
+ "(backward from src2 dominance conflict): "
+ << *DefMI);
+ propagateExclusionForward(DefMI, ExcludedMFMAs);
+ Changed = true;
}
}
- }
+ } while (Changed);
+
return ExcludedMFMAs;
}
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
index 2d81741ec9759..7841edcf8671c 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -48,6 +48,10 @@
entry:
unreachable
}
+ define void @test_src2_exclusion_fixpoint() #0 {
+ entry:
+ unreachable
+ }
attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
...
@@ -820,3 +824,96 @@ body: |
KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
S_ENDPGM 0
...
+
+---
+#
+# Exclusion is a fixpoint: excluding one candidate re-classifies another's src2.
+#
+# bb.0
+# / \
+# bb.1 bb.2 %rA.subN = V_MOV (non-AGPR-form partial def, fragments %rA)
+# \ /
+# bb.3 %resA = MFMA(.., %rA) candidate A, src2=%rA
+# / \
+# bb.4 bb.5 %acc.subN = COPY %rA.subN (partial subreg def; source = A's src2)
+# \ /
+# bb.6 %resB = MFMA(.., %acc) candidate B, src2=%acc
+#
+# A is excluded (non-dominating partial defs fragment %rA), so %rA stays VGPR and
+# the COPYs feeding %acc lose AGPR-form -- B must be excluded too. A single pass
+# misses B and rewrites it, splitting %acc's live interval; the fixpoint
+# re-classification catches B, keeping both MFMAs in vgprcd (VGPR) form.
+#
+# CHECK-LABEL: name: test_src2_exclusion_fixpoint
+# CHECK: %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_exclusion_fixpoint
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %cond:sgpr_32 = COPY $sgpr6
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %rA:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.4:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.6:
+ %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
>From 3dc2a7a69de85a22f8442c7a839590e9d3d12509 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 10:50:03 +0000
Subject: [PATCH 5/5] [AMDGPU] Factor MFMA-form copy analysis into shared
copy-plan analysis
initHeuristics and rewrite duplicated the logic for which src2/dst copies a
rewrite needs. Extract it into an analyzeCopyPoints helper returning an
MFMACopyPlan, so both compute the same plan from the same code and the cost
model and the rewrite agree by construction. Also switch to a post-exclusion
RewriteSet so downstream queries test membership instead of carrying a
separate ExcludedMFMAs set.
analyzeCopyPoints is called once per site (not cached) so each plan reflects
the current MIR, including edits from earlier candidates sharing a dst vreg.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 236 +++++++++-----------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 30 ++-
2 files changed, 136 insertions(+), 130 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index df44f4c525bf3..313671f83c655 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2393,18 +2393,13 @@ static bool isReachingDefAGPRForm(
bool RewriteMFMAFormStage::hasUseRequiringVGPR(
ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallSetVector<MachineInstr *, 16> &RewriteSet,
- const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
for (SlotIndex RDIdx : Src2ReachingDefs) {
const MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- // If RD is a non-excluded MFMA candidate, its dst will be reclassified to
- // AGPR and Case 2 will insert vreg bridge copies for all non-MAI uses of
- // its dst. Those uses therefore do not impose a VGPR constraint on src2.
- // The !ExcludedMFMAs.count(RD) guard is defensive:
- // propagateExclusionForward ensures that if an excluded MFMA's dst is used
- // as src2, the consumer MFMA is also excluded and hasUseRequiringVGPR is
- // never called for it.
- if (TII->isMAI(*RD) && RewriteSet.contains(RD) && !ExcludedMFMAs.count(RD))
+ // A rewritable candidate's dst becomes AGPR and Case 2 bridges its non-MAI
+ // uses, so it imposes no VGPR constraint on src2. Excluded MFMAs are no
+ // longer in RewriteSet.
+ if (TII->isMAI(*RD) && RewriteSet.contains(RD))
continue;
SmallVector<MachineOperand *, 8> ReachingUses;
findReachingUses(RD, DAG.LIS, ReachingUses);
@@ -2412,8 +2407,7 @@ bool RewriteMFMAFormStage::hasUseRequiringVGPR(
const MachineInstr *UseMI = UseMO->getParent();
if (UseMI->isCopy())
continue;
- if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI) &&
- !ExcludedMFMAs.count(UseMI))
+ if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI))
continue;
return true;
}
@@ -2764,39 +2758,91 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
return ExcludedMFMAs;
}
+RewriteMFMAFormStage::MFMACopyPlan RewriteMFMAFormStage::analyzeCopyPoints(
+ MachineInstr *MI, const SmallSetVector<MachineInstr *, 16> &GroupSet,
+ const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR) {
+ MFMACopyPlan Plan;
+
+ // Case 1: the reaching defs of the src2 operand that need a bridge copy.
+ // If src2 stays VGPR, every reaching def needs a copy; otherwise reaching
+ // defs already in AGPR form (group members, av-movs, group copies) don't.
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2->isReg()) {
+ SmallVector<SlotIndex, 8> Src2ReachingDefs;
+ findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
+ for (SlotIndex RDIdx : Src2ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!Src2NeedsVGPR && isReachingDefAGPRForm(RD, GroupSet, Src2Regs, *TII))
+ continue;
+ Plan.Src2DefsNeedingCopy.insert(RD);
+ }
+ }
+
+ // Case 2 and Case 3: the reaching uses of the dst, and the non-MAI reaching
+ // defs of those reaching uses. Group members read the AGPR result directly
+ // and need no copy.
+ SmallVector<MachineOperand *, 8> DstReachingUses;
+ findReachingUses(MI, DAG.LIS, DstReachingUses);
+ for (MachineOperand *RUOp : DstReachingUses) {
+ MachineInstr *UserMI = RUOp->getParent();
+ if (TII->isMAI(*UserMI) && GroupSet.contains(UserMI))
+ continue;
+
+ Plan.DstUsesNeedingCopy.insert(RUOp);
+
+ // Non-rewritten MAI: its defs aren't being reclassified.
+ if (TII->isMAI(*UserMI))
+ continue;
+
+ SmallVector<SlotIndex, 8> DstUsesReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
+ for (SlotIndex RDIndex : DstUsesReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
+ if (TII->isMAI(*RD))
+ continue;
+ Plan.DstUseDefsNeedingCopy.insert(RD);
+ }
+ }
+
+ return Plan;
+}
+
bool RewriteMFMAFormStage::initHeuristics(
std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
bool Changed = false;
- // Pass 1: collect candidate group.
- // RewriteSet/CandSrc2Regs are needed by isReachingDefAGPRForm and
- // hasUseRequiringVGPR; collect them before any setDesc/setRegClass changes.
+ // Pass 1: collect the candidate group (pre-exclusion) before any setDesc/
+ // setRegClass changes; computeExclusionSet needs the complete group.
SmallSetVector<MachineInstr *, 16> RewriteSet;
- DenseSet<Register> CandSrc2Regs;
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
- continue;
- RewriteSet.insert(&MI);
- MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
- }
-
- // Phase 1: identify candidates that cannot be safely rewritten.
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (isRewriteCandidate(&MI))
+ RewriteSet.insert(&MI);
+
+ // Remove unsafe candidates so RewriteSet becomes the post-exclusion group --
+ // both the Pass 2 worklist and the "will be AGPR-form" set queried
+ // downstream.
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs =
computeExclusionSet(RewriteSet);
+ for (MachineInstr *Excluded : ExcludedMFMAs) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *Excluded);
+ RewriteSet.remove(Excluded);
+ }
- // Pass 2: compute heuristics for non-excluded candidates.
+ // src2 regs of the post-exclusion group only: an excluded candidate's src2
+ // stays VGPR, so isReachingDefAGPRForm must not treat a COPY from it as
+ // AGPR-form.
+ DenseSet<Register> CandSrc2Regs;
for (MachineInstr *MI : RewriteSet) {
- if (ExcludedMFMAs.count(MI)) {
- LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *MI);
- continue;
- }
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
+ // Pass 2: compute heuristics for the remaining (rewritable) candidates.
+ for (MachineInstr *MI : RewriteSet) {
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
assert(ReplacementOp != -1);
@@ -2804,21 +2850,22 @@ bool RewriteMFMAFormStage::initHeuristics(
MI->setDesc(TII->get(ReplacementOp));
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+
+ // Cache whether src2 must stay VGPR, then get the copy plan reused by
+ // rewrite().
+ bool Src2NeedsVGPR = false;
if (Src2->isReg()) {
SmallVector<SlotIndex, 8> Src2ReachingDefs;
findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
+ Src2NeedsVGPR = hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet);
+ }
+ Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR =
- hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet, ExcludedMFMAs);
- Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
+ MFMACopyPlan Plan =
+ analyzeCopyPoints(MI, RewriteSet, CandSrc2Regs, Src2NeedsVGPR);
- for (SlotIndex RDIdx : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
- continue;
+ if (Src2->isReg()) {
+ for (MachineInstr *RD : Plan.Src2DefsNeedingCopy) {
// If this reaching def writes into the src2 register itself (possibly
// a partial subreg write), the def will be reclassified together with
// src2 and does not need a bridge copy.
@@ -2831,39 +2878,13 @@ bool RewriteMFMAFormStage::initHeuristics(
}
MachineOperand &Dst = MI->getOperand(0);
- SmallVector<MachineOperand *, 8> DstReachingUses;
-
- findReachingUses(MI, DAG.LIS, DstReachingUses);
-
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
- continue;
-
- // For any user of the result of the MFMA which is not an MFMA, we
- // insert a copy. For a given register, we will only insert one copy
- // per user block.
- CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
-
- if (TII->isMAI(*UserMI))
- continue;
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
-
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
-
- // For any definition of the user of the MFMA which is not an MFMA,
- // we insert a copy. We do this to transform all the reaching defs
- // of this use to AGPR. By doing this, we can insert a copy from
- // AGPR to VGPR at the user rather than after the MFMA.
- CopyForDef.insert(RD);
- }
- }
+ // Case 2: one copy per (user block, register). Case 3: one copy per
+ // non-MAI reaching def of the dst uses.
+ for (MachineOperand *RUOp : Plan.DstUsesNeedingCopy)
+ CopyForUse[RUOp->getParent()->getParent()].insert(RUOp->getReg());
+ for (MachineInstr *RD : Plan.DstUseDefsNeedingCopy)
+ CopyForDef.insert(RD);
// Do the rewrite to allow for updated RP calculation.
const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
@@ -3069,6 +3090,14 @@ bool RewriteMFMAFormStage::rewrite(
continue;
MI->setDesc(TII->get(ReplacementOp));
+ bool Src2NeedsVGPR = Src2NeedsVGPRCache.lookup(MI);
+ // Recompute the plan instead of caching initHeuristics' result: it holds
+ // bare MI/operand pointers, so recomputing against the current MIR reflects
+ // the copies and setReg()s done for earlier candidates that may share a dst
+ // vreg (e.g. non-tied accumulation chains).
+ MFMACopyPlan Plan =
+ analyzeCopyPoints(MI, RewriteCandsSet, RewriteSrc2Regs, Src2NeedsVGPR);
+
// Case 1: insert copies for the reaching defs of the Src2Reg.
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
if (Src2->isReg()) {
@@ -3077,22 +3106,8 @@ bool RewriteMFMAFormStage::rewrite(
return false;
Register MappedReg = Src2->getReg();
- SmallVector<SlotIndex, 8> Src2ReachingDefs;
- findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
- SmallSetVector<MachineInstr *, 8> Src2DefsReplace;
-
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR = Src2NeedsVGPRCache.lookup(MI);
-
- for (SlotIndex RDIndex : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteCandsSet, RewriteSrc2Regs, *TII))
- continue;
-
- Src2DefsReplace.insert(RD);
- }
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace =
+ Plan.Src2DefsNeedingCopy;
if (!Src2DefsReplace.empty()) {
auto RI = RedefMap.find(Src2Reg);
@@ -3149,41 +3164,10 @@ bool RewriteMFMAFormStage::rewrite(
return false;
Register MappedReg = DstReg;
- SmallVector<MachineOperand *, 8> DstReachingUses;
-
- SmallVector<MachineOperand *, 8> DstReachingUseCopies;
- SmallVector<MachineInstr *, 8> DstUseDefsReplace;
-
- findReachingUses(MI, DAG.LIS, DstReachingUses);
-
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteCandsSet.contains(UserMI))
- continue;
-
- // If there is a non mai reaching use, then we need a copy.
- if (find(DstReachingUseCopies, RUOp) == DstReachingUseCopies.end())
- DstReachingUseCopies.push_back(RUOp);
-
- // Non-rewritten MAI: its defs aren't being reclassified.
- if (TII->isMAI(*UserMI))
- continue;
-
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
-
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
-
- // If there is a non mai reaching def of this reaching use, then we will
- // need a copy.
- if (find(DstUseDefsReplace, RD) == DstUseDefsReplace.end())
- DstUseDefsReplace.push_back(RD);
- }
- }
+ const SmallSetVector<MachineOperand *, 8> &DstReachingUseCopies =
+ Plan.DstUsesNeedingCopy;
+ const SmallSetVector<MachineInstr *, 8> &DstUseDefsReplace =
+ Plan.DstUseDefsNeedingCopy;
if (!DstUseDefsReplace.empty()) {
auto RI = RedefMap.find(DstReg);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 5909ba4d28c8c..00fb6ac007346 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -15,6 +15,8 @@
#include "GCNRegPressure.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
@@ -547,13 +549,33 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Returns true if any reaching def of src2 (\p Src2ReachingDefs) has a use
/// that requires the def to stay in VGPR form: any use that is not a COPY and
- /// not a non-excluded MFMA candidate (in \p RewriteSet but not in \p
- /// ExcludedMFMAs). Excluded MFMAs are not being rewritten, so their src2
+ /// not a rewritable MFMA candidate (in \p RewriteSet, which is already
+ /// post-exclusion). Excluded MFMAs are not being rewritten, so their src2
/// cannot be reclassified to AGPR.
bool
hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallSetVector<MachineInstr *, 16> &RewriteSet,
- const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet);
+
+ /// The set of copy insertion points required to rewrite a single MFMA
+ /// candidate, shared by initHeuristics (cost estimate) and rewrite (actual
+ /// insertion) so both agree on exactly which copies are needed.
+ struct MFMACopyPlan {
+ /// Non-AGPR-form reaching defs of src2 that need a bridge COPY (Case 1).
+ SmallSetVector<MachineInstr *, 8> Src2DefsNeedingCopy;
+ /// Non-group reaching uses of the dst that need a copy (Case 2).
+ SmallSetVector<MachineOperand *, 8> DstUsesNeedingCopy;
+ /// Non-MAI reaching defs of those dst uses that need a copy (Case 3).
+ SmallSetVector<MachineInstr *, 8> DstUseDefsNeedingCopy;
+ };
+
+ /// Compute the copy plan for MFMA candidate \p MI. \p GroupSet is the
+ /// post-exclusion rewrite group and \p Src2Regs the candidate src2 registers
+ /// (both used to classify AGPR-form reaching defs); \p Src2NeedsVGPR forces
+ /// every src2 reaching def to be bridged.
+ MFMACopyPlan
+ analyzeCopyPoints(MachineInstr *MI,
+ const SmallSetVector<MachineInstr *, 16> &GroupSet,
+ const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR);
public:
bool initGCNSchedStage() override;
More information about the llvm-commits
mailing list