[llvm] [AMDGPU] Port MFMA-form rewrite refinements onto shared copy-plan analysis (PR #210703)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 21 05:48:35 PDT 2026
https://github.com/xgxanq updated https://github.com/llvm/llvm-project/pull/210703
>From b5505ef17623cf8e48c1d3d19018bbc8120039ad Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Fri, 3 Jul 2026 03:54:44 +0000
Subject: [PATCH 1/6] [AMDGPU] Add safe-guard exclusion for MFMA form rewrite
RewriteMFMAFormStage previously used a coarse isRewriteCandidate check
that rejected any MFMA whose dst had a non-MFMA/non-COPY user. Replace
that with a precise exclusion analysis that screens each candidate for
cases where reclassifying to AGPR form would be illegal, and keeps the
rest rewritable.
findReachingDefs is reworked into a SubRange-aware implementation backed
by collectReachingDefsInRange:
- Subreg uses query the SubRange whose LaneMask fully covers the operand
lanes, falling back to all overlapping SubRanges when no single
SubRange provides full coverage.
- Full-reg uses collect across all SubRanges and deduplicate via a
SmallSet (a full-width def appears in every SubRange but counts once).
- Self-referential defs (src2 == dst MFMA) are skipped in the traversal.
findReachingUses skips implicit operands so that a partial subreg def's
implicit full-reg use (RMW lane preservation) is not treated as a real
consumer, preventing spurious bridge copies.
New exclusion analysis (computeExclusionSet), run before any rewriting:
- hasSrc2BridgeConflict: detects when the src2 bridge-copy model breaks
(a MAI def dominating a non-MAI def, or a src2 use not dominated by any
bridge-copy block), with an early-safe return for parallel candidate
defs.
- hasDstSubregConflict: detects dst subreg writers that cannot be
reclassified to AGPR (non-agnostic writers, or agnostic writers with
non-agnostic orphan uses).
- Exclusion propagates forward along the dst->src2 chain and backward to
MAI reaching-defs of a conflicted src2.
hasUseRequiringVGPR now takes the exclusion set so that excluded MFMAs,
which stay in VGPR form, are correctly treated as VGPR-requiring uses.
initHeuristics is split into a candidate-collection pass and a
heuristics pass over non-excluded candidates.
Add rewrite-mfma-form-safe-guard.mir covering the exclusion branches and
update sched_mfma_rewrite_copies.mir for the new behavior.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 536 ++++++++++++---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 62 +-
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 635 ++++++++++++++++++
.../AMDGPU/sched_mfma_rewrite_copies.mir | 170 +++--
4 files changed, 1232 insertions(+), 171 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4f854beaeebe..e893089412115 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -35,6 +35,7 @@
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/Rematerializer.h"
@@ -1315,48 +1316,108 @@ bool GCNSchedStage::initGCNSchedStage() {
return true;
}
-void RewriteMFMAFormStage::findReachingDefs(
- MachineOperand &UseMO, LiveIntervals *LIS,
- SmallVectorImpl<SlotIndex> &DefIdxs) {
- MachineInstr *UseMI = UseMO.getParent();
- LiveInterval &UseLI = LIS->getInterval(UseMO.getReg());
- VNInfo *VNI = UseLI.getVNInfoAt(LIS->getInstructionIndex(*UseMI));
+static void collectReachingDefsInRange(const LiveRange &QR,
+ MachineBasicBlock *UseMBB,
+ SlotIndex UseIdx,
+ const LiveIntervals *LIS,
+ SmallSet<SlotIndex, 8> &ResultSet) {
+ const VNInfo *VNI = QR.getVNInfoAt(UseIdx);
+ if (!VNI)
+ return;
- // If the def is not a PHI, then it must be the only reaching def.
if (!VNI->isPHIDef()) {
- DefIdxs.push_back(VNI->def);
+ ResultSet.insert(VNI->def);
return;
}
- SmallPtrSet<MachineBasicBlock *, 8> Visited = {UseMI->getParent()};
+ SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<MachineBasicBlock *, 8> Worklist;
-
- // Mark the predecessor blocks for traversal
- for (MachineBasicBlock *PredMBB : UseMI->getParent()->predecessors()) {
- Worklist.push_back(PredMBB);
- Visited.insert(PredMBB);
- }
+ for (MachineBasicBlock *PredMBB : UseMBB->predecessors())
+ if (Visited.insert(PredMBB).second)
+ Worklist.push_back(PredMBB);
while (!Worklist.empty()) {
MachineBasicBlock *CurrMBB = Worklist.pop_back_val();
+ SlotIndex CurrMBBEnd = LIS->getMBBEndIdx(CurrMBB).getPrevSlot();
+ const VNInfo *PredVNI = QR.getVNInfoAt(CurrMBBEnd);
+ if (!PredVNI)
+ continue;
- SlotIndex CurrMBBEnd = LIS->getMBBEndIdx(CurrMBB);
- VNInfo *VNI = UseLI.getVNInfoAt(CurrMBBEnd.getPrevSlot());
-
- MachineBasicBlock *DefMBB = LIS->getMBBFromIndex(VNI->def);
-
- // If there is a def in this block, then add it to the list. This is the
- // reaching def of this path.
- if (!VNI->isPHIDef()) {
- DefIdxs.push_back(VNI->def);
+ if (!PredVNI->isPHIDef()) {
+ // Skip self-referential defs (src2 == dst): the MFMA's own result must
+ // not appear as a reaching def of its own src2 operand.
+ if (SlotIndex::isSameInstr(PredVNI->def, UseIdx))
+ continue;
+ ResultSet.insert(PredVNI->def);
continue;
}
- for (MachineBasicBlock *PredMBB : DefMBB->predecessors()) {
+ MachineBasicBlock *DefMBB = LIS->getMBBFromIndex(PredVNI->def);
+ for (MachineBasicBlock *PredMBB : DefMBB->predecessors())
if (Visited.insert(PredMBB).second)
Worklist.push_back(PredMBB);
+ }
+}
+
+void RewriteMFMAFormStage::findReachingDefs(
+ MachineOperand &UseMO, LiveIntervals *LIS,
+ SmallVectorImpl<SlotIndex> &DefIdxs) {
+ MachineInstr *UseMI = UseMO.getParent();
+ Register UseReg = UseMO.getReg();
+ unsigned UseSubReg = UseMO.getSubReg();
+
+ if (!UseReg.isVirtual() || !LIS->hasInterval(UseReg))
+ return;
+
+ LiveInterval &UseLI = LIS->getInterval(UseReg);
+ SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI);
+
+ // Use a set to deduplicate: a full-reg def appears in every SubRange but
+ // must be inserted into DefIdxs only once.
+ SmallSet<SlotIndex, 8> ResultSet;
+
+ if (UseLI.hasSubRanges()) {
+ const TargetRegisterInfo *TRI = DAG.MRI.getTargetRegisterInfo();
+
+ if (UseSubReg) {
+ // Find the SubRange whose LaneMask fully covers the operand's lanes.
+ // If none does (e.g. register initialised via per-lane subreg writes),
+ // fall back to all overlapping SubRanges to capture every reaching def.
+ LaneBitmask UseLanes = TRI->getSubRegIndexLaneMask(UseSubReg);
+ bool FoundFullCoverage = false;
+ for (LiveInterval::SubRange &SR : UseLI.subranges()) {
+ if ((SR.LaneMask & UseLanes) == UseLanes) {
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ FoundFullCoverage = true;
+ break;
+ }
+ }
+ if (!FoundFullCoverage) {
+ for (LiveInterval::SubRange &SR : UseLI.subranges())
+ if ((SR.LaneMask & UseLanes).any())
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ }
+ } else {
+ // Full-reg use: query every subrange so that partial (subreg) defs on
+ // different lanes are all captured.
+ for (LiveInterval::SubRange &SR : UseLI.subranges())
+ collectReachingDefsInRange(SR, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
+ // If the register has SubRanges but none covers the use point,
+ // LiveIntervals is malformed: a full-width def must appear in at least
+ // one SubRange.
+ assert(!ResultSet.empty() &&
+ "hasSubRanges() but no SubRange live at full-reg use: "
+ "LiveInterval construction is inconsistent");
}
+ } else {
+ collectReachingDefsInRange(UseLI, UseMI->getParent(), UseIdx, LIS,
+ ResultSet);
}
+
+ DefIdxs.append(ResultSet.begin(), ResultSet.end());
}
void RewriteMFMAFormStage::findReachingUses(
@@ -1365,6 +1426,12 @@ void RewriteMFMAFormStage::findReachingUses(
SlotIndex DefIdx = LIS->getInstructionIndex(*DefMI);
for (MachineOperand &UseMO :
DAG.MRI.use_nodbg_operands(DefMI->getOperand(0).getReg())) {
+ // Skip implicit operands: partial subreg defs carry an implicit use of the
+ // full register for RMW lane preservation, not as a real consumer of the
+ // MFMA dst value. Treating them as reaching uses inserts a spurious bridge
+ // copy before the partial def, corrupting MappedReg's live range.
+ if (UseMO.isImplicit())
+ continue;
SmallVector<SlotIndex, 8> ReachingDefIndexes;
findReachingDefs(UseMO, LIS, ReachingDefIndexes);
@@ -2290,12 +2357,29 @@ void GCNSchedStage::modifyRegionSchedule(unsigned RegionIdx,
DAG.Regions[RegionIdx].first = MIOrder.front();
}
+static unsigned getDefSubReg(const MachineInstr &MI, Register Reg) {
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg() == Reg)
+ return MO.getSubReg();
+ return AMDGPU::NoSubRegister;
+}
+
+/// Returns true if \p MI is a class-agnostic subreg writer (COPY or AV_MOV).
+/// These lower to v_accvgpr_write after AGPR reclassification and are legal.
+static bool isAgnosticSubregWriter(const MachineInstr *MI) {
+ if (MI->isCopy())
+ return true;
+ unsigned Opc = MI->getOpcode();
+ return Opc == AMDGPU::AV_MOV_B32_IMM_PSEUDO ||
+ Opc == AMDGPU::AV_MOV_B64_IMM_PSEUDO;
+}
+
/// Returns true if reaching def \p RD will be in AGPR form after the rewrite
/// and so needs no bridge copy: a candidate MFMA in \p RewriteSet, an
/// AV_MOV_*_IMM_PSEUDO, or a copy from a candidate src2 reg in \p CandSrc2Regs.
/// A non-candidate MFMA stays in VGPR form and still needs a bridge.
static bool isReachingDefAGPRForm(
- MachineInstr *RD, const SmallPtrSetImpl<MachineInstr *> &RewriteSet,
+ MachineInstr *RD, const SmallSetVector<MachineInstr *, 16> &RewriteSet,
const DenseSet<Register> &CandSrc2Regs, const SIInstrInfo &TII) {
if (TII.isMAI(*RD))
return RewriteSet.contains(RD);
@@ -2309,16 +2393,27 @@ static bool isReachingDefAGPRForm(
bool RewriteMFMAFormStage::hasUseRequiringVGPR(
ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallPtrSetImpl<MachineInstr *> &RewriteSet) {
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
for (SlotIndex RDIdx : Src2ReachingDefs) {
const MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ // If RD is a non-excluded MFMA candidate, its dst will be reclassified to
+ // AGPR and Case 2 will insert vreg bridge copies for all non-MAI uses of
+ // its dst. Those uses therefore do not impose a VGPR constraint on src2.
+ // The !ExcludedMFMAs.count(RD) guard is defensive:
+ // propagateExclusionForward ensures that if an excluded MFMA's dst is used
+ // as src2, the consumer MFMA is also excluded and hasUseRequiringVGPR is
+ // never called for it.
+ if (TII->isMAI(*RD) && RewriteSet.contains(RD) && !ExcludedMFMAs.count(RD))
+ continue;
SmallVector<MachineOperand *, 8> ReachingUses;
findReachingUses(RD, DAG.LIS, ReachingUses);
for (const MachineOperand *UseMO : ReachingUses) {
const MachineInstr *UseMI = UseMO->getParent();
if (UseMI->isCopy())
continue;
- if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI))
+ if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI) &&
+ !ExcludedMFMAs.count(UseMI))
continue;
return true;
}
@@ -2354,24 +2449,247 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
return false;
if (AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode()) == -1)
return false;
- // Reject candidates whose users force an unavoidable bridge copy.
- Register DstReg = MI->getOperand(0).getReg();
- for (const MachineOperand &Use : DAG.MRI.use_nodbg_operands(DstReg)) {
- if (!TII->isMAI(*Use.getParent()) && !Use.getParent()->isCopy())
- return false;
- }
return true;
}
+bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
+ Register Src2Reg) const {
+ auto &MDT = DAG.LIS->getDomTree();
+
+ SmallVector<MachineInstr *, 8> MAIMIs;
+ SmallPtrSet<MachineBasicBlock *, 8> BridgeCopyBlocks; // non-MAI def blocks
+
+ for (SlotIndex SI : DefIdxs) {
+ MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
+ if (TII->isMFMA(*MI))
+ MAIMIs.push_back(MI);
+ else
+ BridgeCopyBlocks.insert(MI->getParent());
+ }
+
+ if (BridgeCopyBlocks.empty())
+ return false; // All defs are MAI; no bridge copies needed.
+
+ // Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
+ // The MFMA first writes all lanes (AGPR), then a non-MAI instruction
+ // partially overwrites some lanes. A bridge copy at the non-MAI def
+ // would read an already-AGPR register and must partially update it —
+ // a read-modify-write that the bridge-copy mechanism cannot implement.
+ if (!MAIMIs.empty()) {
+ SmallVector<MachineInstr *, 8> NonMAIMIs;
+ for (SlotIndex SI : DefIdxs) {
+ MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
+ if (!TII->isMFMA(*MI))
+ NonMAIMIs.push_back(MI);
+ }
+
+ for (MachineInstr *M : MAIMIs)
+ for (MachineInstr *N : NonMAIMIs)
+ if (MDT.dominates(M, N))
+ return true;
+
+ // Early-safe return: if every (MAI, non-MAI) pair is parallel and every
+ // MAI def is itself a rewrite candidate, the rewrite is safe without
+ // invoking Check 2.
+ if (!NonMAIMIs.empty()) {
+ bool AllParallelAndCandidates = true;
+ for (MachineInstr *M : MAIMIs) {
+ if (!isRewriteCandidate(M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ for (MachineInstr *N : NonMAIMIs) {
+ if (MDT.dominates(N, M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ }
+ if (!AllParallelAndCandidates)
+ break;
+ }
+ if (AllParallelAndCandidates)
+ return false;
+ }
+ }
+
+ // Check 2: every use of Src2Reg must be dominated by at least one
+ // bridge-copy block; otherwise %MappedReg would be undefined on some path.
+ //
+ // use_nodbg_operands is used here (not findReachingUses) because rewrite()
+ // replaces the src2 operand of the MFMA with %MappedReg uniformly — it does
+ // not distinguish which reaching def flows to which use. %MappedReg is
+ // defined only in bridge-copy blocks (after non-MAI defs); if any use of
+ // Src2Reg is in a block not dominated by any bridge-copy block, %MappedReg
+ // would be undefined on that path regardless of whether the use is reached
+ // by a MAI or non-MAI def. findReachingUses(non-MAI RD) would miss uses
+ // that arrive only via MAI defs or other defs (e.g. IMPLICIT_DEF on a
+ // bypass path), causing a false negative and silent undefined-read.
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
+ bool Covered = any_of(BridgeCopyBlocks, [&](const MachineBasicBlock *B) {
+ return MDT.dominates(B, UseBlock);
+ });
+ if (!Covered)
+ return true;
+ }
+
+ return false;
+}
+
+void RewriteMFMAFormStage::propagateExclusionForward(
+ MachineInstr *Root, SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
+ SmallVector<MachineInstr *, 8> Worklist = {Root};
+ while (!Worklist.empty()) {
+ MachineInstr *ExclMI = Worklist.pop_back_val();
+ MachineOperand &DstMO = ExclMI->getOperand(0);
+ if (!DstMO.isReg() || !DstMO.getReg().isVirtual())
+ continue;
+ Register DstReg = DstMO.getReg();
+ for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(DstReg)) {
+ MachineInstr *UserMI = UseMO.getParent();
+ if (!isRewriteCandidate(UserMI))
+ continue;
+ MachineOperand *UserSrc2 =
+ TII->getNamedOperand(*UserMI, AMDGPU::OpName::src2);
+ if (!UserSrc2 || !UserSrc2->isReg() || UserSrc2->getReg() != DstReg)
+ continue;
+ if (ExcludedMFMAs.insert(UserMI).second) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] exclude downstream MFMA "
+ "(src2 = excluded MFMA dst): "
+ << *UserMI);
+ Worklist.push_back(UserMI);
+ }
+ }
+ }
+}
+
+// rewrite()'s design principle: reclassify DstReg from VGPR to AGPR class so
+// that the MFMA emits its result directly into AGPR, eliminating the need for
+// a post-MFMA VGPR→AGPR copy. For this reclassification to be legal, every
+// def and every use of DstReg throughout its live range must support
+// AGPR-class operands. hasDstSubregConflict checks this before rewriting:
+//
+bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
+ MachineInstr *MFMA) {
+ SmallVector<MachineOperand *, 8> DstReachingUses;
+ findReachingUses(MFMA, DAG.LIS, DstReachingUses);
+ SmallPtrSet<MachineInstr *, 8> CheckedDefs;
+ SmallVector<MachineInstr *, 4> SafeSubregDefs;
+
+ // Phase 1: classify subreg reaching defs.
+ // Non-agnostic subreg def → immediate conflict.
+ // Agnostic (COPY/AV_MOV) subreg def → defer to orphan-use check.
+ for (MachineOperand *RUOp : DstReachingUses) {
+ SmallVector<SlotIndex, 8> ReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, ReachingDefs);
+ for (SlotIndex RDIdx : ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!CheckedDefs.insert(RD).second)
+ continue;
+ if (getDefSubReg(*RD, DstReg) == AMDGPU::NoSubRegister || TII->isMAI(*RD))
+ continue;
+ if (!isAgnosticSubregWriter(RD))
+ return true;
+ SafeSubregDefs.push_back(RD);
+ }
+ }
+
+ // Phase 2: agnostic subreg defs lower to v_accvgpr_write after reclassify,
+ // so their orphan uses read an AGPR sub-register. Only COPY and AV_MOV can
+ // legally source an AGPR sub-register; anything else is a conflict.
+ for (MachineInstr *RD : SafeSubregDefs) {
+ SlotIndex RDIdx = DAG.LIS->getInstructionIndex(*RD);
+ for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(DstReg)) {
+ if (UseMO.isImplicit() || TII->isMAI(*UseMO.getParent()))
+ continue;
+ SmallVector<SlotIndex, 8> UseReachingDefs;
+ findReachingDefs(UseMO, DAG.LIS, UseReachingDefs);
+ if (any_of(UseReachingDefs,
+ [RDIdx](SlotIndex SI) {
+ return SlotIndex::isSameInstr(SI, RDIdx);
+ }) &&
+ !isAgnosticSubregWriter(UseMO.getParent()))
+ return true;
+ }
+ }
+ return false;
+}
+
+SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
+ // Per-MI checks run cheapest-first:
+ // 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
+ // a read-modify-write on AGPR (MAI def dominates non-MAI def), or
+ // absent on some CFG path to a src2 use.
+ // 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
+ // be reclassified to AGPR. Skipped when check 1 already forces
+ // exclusion (!HasConflict &&).
+ // Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
+ // and backward (MAI reaching-defs of a conflicted src2).
+ SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ Register DstReg = MI->getOperand(0).getReg();
+
+ bool HasConflict = false;
+ SmallVector<SlotIndex, 8> Src2Defs;
+ if (Src2->isReg()) {
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ LLVM_DEBUG({
+ dbgs() << "[computeExclusionSet] candidate: " << *MI;
+ dbgs() << " src2 reaching defs (" << Src2Defs.size() << "):\n";
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *D = DAG.LIS->getInstructionFromIndex(SI);
+ dbgs() << " " << SI << " opcode=" << (D ? (int)D->getOpcode() : -1)
+ << "\n";
+ if (D)
+ dbgs() << " " << *D;
+ }
+ });
+ HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg());
+ }
+ bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+
+ if (!HasConflict && !HasDstSubregDef)
+ continue;
+
+ if (ExcludedMFMAs.insert(MI).second) {
+ LLVM_DEBUG(
+ dbgs() << "[computeExclusionSet] exclude MFMA ("
+ << (HasConflict ? "src2 dominance conflict" : "")
+ << (HasConflict && HasDstSubregDef ? " + " : "")
+ << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
+ << "): " << *MI);
+ propagateExclusionForward(MI, ExcludedMFMAs);
+ }
+
+ // Backward: exclude MAI reaching-defs that are themselves candidates.
+ if (HasConflict) {
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
+ if (TII->isMFMA(*DefMI) && isRewriteCandidate(DefMI) &&
+ ExcludedMFMAs.insert(DefMI).second) {
+ LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
+ "(backward from src2 dominance conflict): "
+ << *DefMI);
+ propagateExclusionForward(DefMI, ExcludedMFMAs);
+ }
+ }
+ }
+ }
+ return ExcludedMFMAs;
+}
+
bool RewriteMFMAFormStage::initHeuristics(
std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
bool Changed = false;
- // Collect the candidate group, its members share AGPR-form operands
- // post-rewrite, so reaching defs feeding any member don't need bridge copy.
- SmallPtrSet<MachineInstr *, 16> RewriteSet;
+ // Pass 1: collect candidate group.
+ // RewriteSet/CandSrc2Regs are needed by isReachingDefAGPRForm and
+ // hasUseRequiringVGPR; collect them before any setDesc/setRegClass changes.
+ SmallSetVector<MachineInstr *, 16> RewriteSet;
DenseSet<Register> CandSrc2Regs;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
@@ -2384,86 +2702,98 @@ bool RewriteMFMAFormStage::initHeuristics(
}
}
- // Prepare for the heuristics
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
- continue;
+ // Phase 1: identify candidates that cannot be safely rewritten.
+ SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs =
+ computeExclusionSet(RewriteSet);
- int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
- assert(ReplacementOp != -1);
+ // Pass 2: compute heuristics for non-excluded candidates.
+ for (MachineInstr *MI : RewriteSet) {
+ if (ExcludedMFMAs.count(MI)) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *MI);
+ continue;
+ }
- RewriteCands.push_back({&MI, MI.getOpcode()});
- MI.setDesc(TII->get(ReplacementOp));
+ int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
+ assert(ReplacementOp != -1);
- MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (Src2->isReg()) {
- SmallVector<SlotIndex, 8> Src2ReachingDefs;
- findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
-
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR = hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet);
- Src2NeedsVGPRCache[&MI] = Src2NeedsVGPR;
-
- for (SlotIndex RDIdx : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
- continue;
- CopyForDef.insert(RD);
- }
- }
+ RewriteCands.push_back({MI, MI->getOpcode()});
+ MI->setDesc(TII->get(ReplacementOp));
- MachineOperand &Dst = MI.getOperand(0);
- SmallVector<MachineOperand *, 8> DstReachingUses;
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2->isReg()) {
+ SmallVector<SlotIndex, 8> Src2ReachingDefs;
+ findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
- findReachingUses(&MI, DAG.LIS, DstReachingUses);
+ // If src2 has a use that must remain VGPR, it cannot be reclassified to
+ // AGPR.
+ bool Src2NeedsVGPR =
+ hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet, ExcludedMFMAs);
+ Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
+ for (SlotIndex RDIdx : Src2ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!Src2NeedsVGPR &&
+ isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
continue;
+ // If this reaching def writes into the src2 register itself (possibly
+ // a partial subreg write), the def will be reclassified together with
+ // src2 and does not need a bridge copy.
+ if (any_of(RD->defs(), [&](const MachineOperand &DefOp) {
+ return DefOp.isReg() && DefOp.getReg() == Src2->getReg();
+ }))
+ continue;
+ CopyForDef.insert(RD);
+ }
+ }
+
+ MachineOperand &Dst = MI->getOperand(0);
+ SmallVector<MachineOperand *, 8> DstReachingUses;
- // For any user of the result of the MFMA which is not an MFMA, we
- // insert a copy. For a given register, we will only insert one copy
- // per user block.
- CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
+ findReachingUses(MI, DAG.LIS, DstReachingUses);
- if (TII->isMAI(*UserMI))
- continue;
+ for (MachineOperand *RUOp : DstReachingUses) {
+ MachineInstr *UserMI = RUOp->getParent();
+ // Group members read the AGPR result directly.
+ if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
+ continue;
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
+ // For any user of the result of the MFMA which is not an MFMA, we
+ // insert a copy. For a given register, we will only insert one copy
+ // per user block.
+ CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
+ if (TII->isMAI(*UserMI))
+ continue;
- // For any definition of the user of the MFMA which is not an MFMA,
- // we insert a copy. We do this to transform all the reaching defs
- // of this use to AGPR. By doing this, we can insert a copy from
- // AGPR to VGPR at the user rather than after the MFMA.
- CopyForDef.insert(RD);
- }
- }
+ SmallVector<SlotIndex, 8> DstUsesReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
- // Do the rewrite to allow for updated RP calculation.
- const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
- const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
- DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
- if (Src2->isReg()) {
- // Have to get src types separately since subregs may cause C and D
- // registers to be different types even though the actual operand is
- // the same size.
- const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
- DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ for (SlotIndex RDIndex : DstUsesReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
+ if (TII->isMAI(*RD))
+ continue;
+
+ // For any definition of the user of the MFMA which is not an MFMA,
+ // we insert a copy. We do this to transform all the reaching defs
+ // of this use to AGPR. By doing this, we can insert a copy from
+ // AGPR to VGPR at the user rather than after the MFMA.
+ CopyForDef.insert(RD);
}
- Changed = true;
}
+
+ // Do the rewrite to allow for updated RP calculation.
+ const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
+ const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
+ DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
+ if (Src2->isReg()) {
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ }
+ Changed = true;
}
return Changed;
@@ -2640,7 +2970,7 @@ bool RewriteMFMAFormStage::rewrite(
// Collect the candidate group; its members share AGPR-form operands
// post-rewrite, so reaching defs feeding any member need no bridge copy.
- SmallPtrSet<MachineInstr *, 16> RewriteCandsSet;
+ SmallSetVector<MachineInstr *, 16> RewriteCandsSet;
DenseSet<Register> RewriteSrc2Regs;
for (auto &[MI, OriginalOpcode] : RewriteCands) {
RewriteCandsSet.insert(MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2059f4e6479ff..25a696f3e540b 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -472,6 +472,54 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// \returns true if this MI is a rewrite candidate.
bool isRewriteCandidate(MachineInstr *MI) const;
+ /// Returns true if the src2 reaching defs \p DefIdxs of \p Src2Reg have a
+ /// conflict that prevents safe bridge-copy insertion after non-MAI defs.
+ ///
+ /// rewrite()'s design principle for src2: instead of reclassifying Src2Reg
+ /// to AGPR directly, Case 1 creates a fresh VGPR %MappedReg and inserts a
+ /// bridge COPY (Src2Reg → %MappedReg) after each non-MAI reaching def. The
+ /// MFMA's src2 operand is then replaced with %MappedReg, which rewrite()
+ /// subsequently reclassifies to AGPR. This preserves Src2Reg's VGPR class
+ /// for non-MFMA users while giving the MFMA an AGPR-class src2.
+ ///
+ /// For this strategy to be correct, every bridge COPY must produce a
+ /// well-defined %MappedReg at every src2 use site. Two conditions break
+ /// this requirement:
+ /// Check 1: if a MAI def dominates a non-MAI def, the bridge copy after the
+ /// non-MAI def would need to partially update an already-AGPR register —
+ /// illegal for rewrite()'s COPY model.
+ /// Early-safe return: if all (MAI, non-MAI) pairs are parallel in the CFG
+ /// and every MAI def is itself a candidate, the non-MAI defs can be bridged
+ /// without conflict; Check 2 is skipped.
+ /// Check 2: every use of \p Src2Reg must be dominated by at least one
+ /// non-MAI def block (where a bridge copy is inserted); otherwise %MappedReg
+ /// is undefined on some CFG path to a use.
+ bool hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
+ Register Src2Reg) const;
+
+ /// Returns true if reclassifying \p DstReg to AGPR would require bridge
+ /// copies beyond rewrite()'s capability (full-register copies only).
+ /// Phase 1: non-agnostic subreg writer (e.g. V_MOV) cannot write an AGPR
+ /// sub-register lane → conflict. Agnostic writers (COPY/AV_MOV) lower to
+ /// v_accvgpr_write and are legal but their orphan uses are checked in
+ /// Phase 2. Phase 2: if an orphan use of an agnostic subreg def is
+ /// non-agnostic, it cannot legally source an AGPR sub-register lane →
+ /// conflict.
+ bool hasDstSubregConflict(Register DstReg, MachineInstr *MFMA);
+
+ /// Transitively exclude from \p ExcludedMFMAs any rewrite candidate whose
+ /// src2 is the dst of \p Root or of any already-excluded MFMA.
+ void
+ propagateExclusionForward(MachineInstr *Root,
+ SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
+
+ /// Compute the set of rewrite candidates in \p RewriteSet that must be
+ /// excluded from rewriting due to src2 dominance conflicts or dst subreg
+ /// conflicts. Exclusion propagates forward (dst→src2 chain) and backward
+ /// (MAI reaching-defs of a conflicted src2). No IR is mutated.
+ SmallPtrSet<MachineInstr *, 16>
+ computeExclusionSet(const SmallSetVector<MachineInstr *, 16> &RewriteSet);
+
/// Resets all candidates in \p RewriteCands back to VGPR form.
void resetRewriteCandsToVGPR(
ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
@@ -486,11 +534,15 @@ class RewriteMFMAFormStage : public GCNSchedStage {
void findReachingUses(const MachineInstr *DefMI, LiveIntervals *LIS,
SmallVectorImpl<MachineOperand *> &ReachingUses);
- /// Returns true if the src2 register with reaching defs \p Src2ReachingDefs
- /// has a use other than a group MFMA (in \p RewriteSet) or a copy, which
- /// would keep it in VGPR form rather than let it be reclassified to AGPR.
- bool hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
+ /// Returns true if any reaching def of src2 (\p Src2ReachingDefs) has a use
+ /// that requires the def to stay in VGPR form: any use that is not a COPY and
+ /// not a non-excluded MFMA candidate (in \p RewriteSet but not in \p
+ /// ExcludedMFMAs). Excluded MFMAs are not being rewritten, so their src2
+ /// cannot be reclassified to AGPR.
+ bool
+ hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
public:
bool initGCNSchedStage() override;
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
new file mode 100644
index 0000000000000..58a1ca4c8dd04
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -0,0 +1,635 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: %s -o - | FileCheck %s
+
+--- |
+ define void @test_src2_backedge_bypass_excluded() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_subreg_overwrite_cross_bb() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_avmov_agprform_rewrite() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_buffer_load_early_safe_rewrite() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_selfref_avmov_init() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_dst_subreg_loop_copy_consumer() #0 {
+ entry:
+ br label %loop
+ loop:
+ br i1 undef, label %loop, label %exit
+ exit:
+ ret void
+ }
+ define void @test_dst_subreg_copy_valu_consumer() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_partial_exclude_avmov_src2() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+#
+# Loop with bypass path triggers hasSrc2BridgeConflict.
+#
+# CFG:
+# bb.0 ──► bb.1 (preheader) ──► bb.2 (loop) ──► bb.3 (bridge) ──► (ret)
+# (entry) ↑___________|
+# │ (back-edge)
+# └────────────────────────────────────────────► bb.3 (bypass)
+#
+# %acc reaching defs for MFMA_A src2: {AV_MOV×4 (bb.1, non-MAI), MFMA_C (bb.2, MAI)}.
+# hasSrc2BridgeConflict: bb.1 does not dominate bb.3 (bypass skips bb.1) → excluded.
+# MFMA_C excluded by backward propagation; MFMA_B by forward propagation.
+# All three loop MFMAs stay in vreg form.
+#
+# Without exclusion: %acc reclassified to AGPR; the AV_MOV subreg inits in bb.1
+# have no bridge-copy reader and are DCE'd to dead undef. On the first iteration
+# (bb.0→bb.1→bb.2) MFMA_A reads the IMPLICIT_DEF garbage from bb.0 instead of
+# the intended initial accumulator → silent wrong result on iteration 1.
+#
+# CHECK-LABEL: name: test_src2_backedge_bypass_excluded
+# CHECK: bb.2:
+# CHECK: %r1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: V_MFMA_F32_16X16X4F32_e64
+name: test_src2_backedge_bypass_excluded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs > gfx90a limit 256 → excess pressure.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ ; IMPLICIT_DEF covers %acc on the bypass path (bb.0→bb.3) so the verifier
+ ; is satisfied. bb.1's AV_MOVs kill this def before bb.2, so it does not
+ ; appear in findReachingDefs at MFMA_A and is absent from BridgeCopyBlocks.
+ %acc:vreg_128_align2 = IMPLICIT_DEF
+
+ SCHED_BARRIER 0
+
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ ; Subreg init of %acc (AV_MOV, non-MAI). Creates SubRanges on %acc:
+ ; findReachingDefs returns these as non-MAI reaching defs for MFMA_A src2=%acc.
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+
+ ; MFMA_A: src2=%acc — hasSrc2BridgeConflict fires: bb.1 not dom bb.3 (bypass).
+ %r1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_B: excluded by forward propagation from MFMA_A.
+ %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %r1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_C: redefines %acc (MAI, back-edge def). Excluded by backward propagation.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %r2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ bb.3:
+ ; Explicit use of %acc: hasSrc2BridgeConflict sees this use in bb.3, which is
+ ; not dominated by bb.1 (preheader) due to the bypass path → conflict.
+ %bridge_use:vreg_128_align2 = COPY %acc:vreg_128_align2
+
+ KILL %bridge_use
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+
+---
+#
+# Test: diamond CFG with non-MAI subreg overwrites of MFMA dst blocks rewrite.
+#
+# CFG (diamond):
+#
+# bb.0 ← %acc = MFMA(...)
+# / \
+# bb.1 bb.2 ← bb.1: %acc.sub0 = V_MOV 1 (non-MAI subreg overwrite)
+# \ / bb.2: %acc.sub1 = V_MOV 2 (non-MAI subreg overwrite)
+# bb.3 ← %res = MFMA(..., %acc), %res2 = MFMA(..., %res)
+#
+# %acc dst has non-MAI subreg overwrites (V_MOV in bb.1/bb.2) → excluded.
+# %res/%res2 excluded by propagation (src2 = excluded dst).
+#
+# Primary coverage: computeExclusionSet HasDstSubregDef path — %acc's MFMA has
+# an immediate src2 so hasSrc2BridgeConflict never runs; hasDstSubregConflict is
+# the sole trigger and the root of the entire exclusion chain.
+#
+# Without exclusion: %acc reclassified to AGPR → bridge COPYs inserted in both
+# bb.1 and bb.2 define the same vreg without a PHI; MachineVerifier aborts with
+# "Virtual register defs don't dominate all uses" (dominator violation crash).
+#
+# CHECK-LABEL: name: test_subreg_overwrite_cross_bb
+# CHECK: bb.0:
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: bb.3:
+# CHECK: %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+name: test_subreg_overwrite_cross_bb
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ ; MAI full-reg def of %acc (all 4 lanes) — contributes MAI to ResultSet.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ SCHED_BARRIER 0
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ ; bb.1: non-MAI sub0 overwrite — contributes non-MAI to ResultSet.
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ %acc.sub0:vreg_128_align2 = V_MOV_B32_e32 1, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; bb.2: non-MAI sub1 overwrite — contributes non-MAI to ResultSet.
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc.sub1:vreg_128_align2 = V_MOV_B32_e32 2, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; bb.3: %res/%res2 already excluded by forward propagation from %acc's MFMA.
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; Chain the result into another MFMA (MAI use of %res -> CopyForUse = 0).
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+
+---
+#
+# Diamond CFG where hasSrc2BridgeConflict passes.
+#
+# CFG (diamond):
+#
+# bb.0 ← %acc.subN = AV_MOV_B32_IMM_PSEUDO 0 (per-lane AGPR-form init)
+# / \
+# bb.1 bb.2 ← bb.1: %acc live-through (no def)
+# \ / bb.2: %acc = MFMA_prev(a, b, 0) [MAI]
+# bb.3 ← %res = MFMA_target(a, b, %acc), %res2 = MFMA_chain(a, b, %res)
+#
+# reaching defs for %res src2=%acc: {AV_MOV(bb.0, non-MAI), MFMA_prev(bb.2, MAI)}.
+# hasSrc2BridgeConflict: Check1 NO (bb.2 does not dominate bb.0),
+# Check2 NO (bb.0 dominates bb.3 → bridge covered) → not excluded.
+# AV_MOV and MFMA_prev both AGPR-form → CopyCost=0 → rewrite fires.
+#
+# CHECK-LABEL: name: test_src2_avmov_agprform_rewrite
+# CHECK-NOT: COPY
+# CHECK: bb.2:
+# CHECK: %acc:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: {{.*}}:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_avmov_agprform_rewrite
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %p9:vreg_1024 = IMPLICIT_DEF
+ %p10:vreg_1024 = IMPLICIT_DEF
+ %p11:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %p9, %p10, %p11
+
+ S_ENDPGM 0
+
+---
+#
+# Test: RewriteMFMAFormStage — hasSrc2BridgeConflict early-safe return with diamond CFG.
+#
+# CFG (diamond):
+#
+# bb.0 (entry: pressure fill + branch)
+# / \
+# bb.1 bb.2 ← bb.1: %acc = BUFFER_LOAD (non-MAI)
+# \ / bb.2: %acc = MFMA_prev(a, b, 0) (MAI, src2=imm → candidate)
+# bb.3 ← %res = MFMA_target(a, b, %acc), %res2 = MFMA_chain(a, b, %res)
+#
+# hasSrc2BridgeConflict for MFMA_target: reaching defs = {BUFFER_LOAD, MFMA_prev}.
+# Check 1: MFMA_prev(bb.2) does not dominate BUFFER_LOAD(bb.1) → false.
+# Early-safe return: MFMA_prev is a candidate and BUFFER_LOAD(bb.1) does not
+# dominate MFMA_prev(bb.2) → AllParallelAndCandidates=true → return false;
+# Check 2 not reached. All three MFMAs are rewrite candidates.
+#
+# 9 × vreg_1024 = 288 ArchVGPRs > 256 limit → ExcessArchVGPR fires; rewrite fires.
+# Bridge copy inserted in bb.1 after BUFFER_LOAD.
+# CHECK-LABEL: name: test_src2_buffer_load_early_safe_rewrite
+# CHECK: bb.1:
+# CHECK: %acc:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET
+# CHECK: {{.*}}:areg_128_align2 = COPY %acc
+# CHECK: bb.2:
+# CHECK: {{.*}}:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK-NOT: vgprcd
+name: test_src2_buffer_load_early_safe_rewrite
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 × vreg_1024 = 288 ArchVGPRs (> gfx90a limit 256); KILL in bb.3 keeps them live.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %rsrc:sgpr_128 = IMPLICIT_DEF
+ %soff:sgpr_32 = IMPLICIT_DEF
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET %rsrc:sgpr_128, %soff:sgpr_32, 0, 0, 0, implicit $exec :: (load (s128))
+
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+
+---
+#
+# Self-referential MFMA (dst == src2) with subreg AV_MOV init in entry block.
+#
+# CFG:
+# bb.0 (entry) ──► bb.1 (loop) ──► bb.2 (exit)
+# ↑___________|
+# (back-edge)
+#
+# src2=%acc reaching defs: {AV_MOV×4 (bb.0, non-MAI), MFMA_A back-edge (MAI)}.
+# hasSrc2BridgeConflict: bb.0 dominates bb.1 → Check 2 covered → false.
+# AV_MOV and MFMA_A both AGPR-form → no bridge copy. Both MFMAs rewritten.
+#
+# CHECK-LABEL: name: test_src2_selfref_avmov_init
+# CHECK-NOT: COPY
+# CHECK: bb.1:
+# CHECK: %acc:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: %r2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_selfref_avmov_init
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0, $sgpr4_sgpr5
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ SCHED_BARRIER 0
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7e000000), %bb.2(0x02000000)
+
+ ; MFMA_A: self-ref (dst=%acc == src2=%acc). Back-edge def is AGPR-form.
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; MFMA_B: src2=%acc is MFMA_A output (AGPR-form) → no bridge copy.
+ %r2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ KILL %r2
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+
+---
+#
+# Test: MFMA with non-MAI subreg overwrite on dst is excluded from rewrite;
+# sibling MFMA without subreg overwrite is rewritten.
+#
+# %dst1: %dst1.sub0 = V_MOV in same BB after MFMA → excluded (stays vgprcd).
+# Without exclusion: %dst1 reclassified to AGPR; V_MOV still writes vreg sub0,
+# then a bridge COPY reads the whole %dst1:vreg with mixed AGPR/VGPR lanes →
+# wrong result (sub1~sub3 sourced from AGPR, sub0 from VGPR V_MOV partial write).
+# %dst0: no subreg overwrite → rewritten to AGPR form.
+#
+# CHECK-LABEL: name: test_dst_subreg_loop_copy_consumer
+# CHECK: bb.1.loop:
+# CHECK: V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %dst1.sub0:vreg_128_align2 = V_MOV_B32_e32 1
+# CHECK: V_MFMA_F32_16X16X4F32_e64
+name: test_dst_subreg_loop_copy_consumer
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ successors: %bb.1(0x80000000)
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ %f0:vgpr_32 = IMPLICIT_DEF
+ %f1:vgpr_32 = IMPLICIT_DEF
+ %f2:vgpr_32 = IMPLICIT_DEF
+ %f3:vgpr_32 = IMPLICIT_DEF
+ undef %src2.sub0:vreg_128_align2 = COPY %f0:vgpr_32
+ %src2.sub1:vreg_128_align2 = COPY %f1:vgpr_32
+ %src2.sub2:vreg_128_align2 = COPY %f2:vgpr_32
+ %src2.sub3:vreg_128_align2 = COPY %f3:vgpr_32
+
+ S_BRANCH %bb.1
+
+ bb.1.loop:
+ successors: %bb.1(0x78000000), %bb.2(0x08000000)
+
+ SCHED_BARRIER 0
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1.sub0:vreg_128_align2 = V_MOV_B32_e32 1, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ $scc = IMPLICIT_DEF
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.2.exit:
+ %tmp0:vreg_128_align2 = COPY %dst0:vreg_128_align2
+ %tmp1:vreg_128_align2 = COPY %dst1:vreg_128_align2
+ S_ENDPGM 0, implicit %tmp0, implicit %tmp1
+
+---
+#
+# Test: hasDstSubregConflict Phase 2 — MFMA dst subreg written by COPY (agnostic,
+# Phase 1 safe) but read by V_MUL (non-agnostic orphan use) → excluded.
+# propagateExclusionForward excludes downstream chain c0..c3; no rewrite.
+#
+# Without exclusion: %mfma reclassified to AGPR; COPY %mfma.sub2 lowers to
+# v_accvgpr_read, but V_MUL reads the sub-register as VGPR — the bridge COPY
+# inserted after the agnostic subreg def is not jointly dominated by all defs
+# of %mfma → "Use not jointly dominated by defs" LIS crash.
+# CHECK-LABEL: name: test_dst_subreg_copy_valu_consumer
+# CHECK: %mfma:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %mfma.sub2:vreg_128_align2 = COPY %val
+# CHECK: %c0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %c3:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+name: test_dst_subreg_copy_valu_consumer
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %val:vgpr_32 = IMPLICIT_DEF
+
+ %mfma:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ %mfma.sub2:vreg_128_align2 = COPY %val:vgpr_32
+
+ %result:vgpr_32 = nofpexcept V_MUL_F32_e32 0, %mfma.sub2:vreg_128_align2, implicit $mode, implicit $exec
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7e000000), %bb.2(0x02000000)
+
+ %c0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %mfma:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c0:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %c3:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %c2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ KILL %c3, %result
+ S_ENDPGM 0
+
+---
+#
+# Test: RewriteMFMAFormStage — hasUseRequiringVGPR uses pre-exclusion RewriteSet,
+# leading to %acc being directly reclassified to AGPR without a bridge copy for
+# the excluded MFMA_B.
+#
+# Bug trigger path:
+# %acc is initialized via AV_MOV_B32_IMM_PSEUDO subreg writes (AGPR-form).
+# No candidate MFMA writes %acc, so Case 2 (dst-use bridge copies) never
+# processes %acc — the only bridge-copy opportunity is Case 1 (src2 bridge).
+#
+# hasUseRequiringVGPR({AV_MOV×4}, RewriteSet):
+# findReachingUses(AV_MOV.subN) → {MFMA_A, MFMA_B}.
+# MFMA_B: isMAI && in pre-exclusion RewriteSet → "safe" → returns false.
+# → Src2NeedsVGPR=false (WRONG: MFMA_B is excluded and stays VGPR form).
+#
+# isReachingDefAGPRForm(AV_MOV) = true → Src2DefsReplace={}.
+# → %acc directly reclassified to AGPR.
+# → No bridge copy inserted for MFMA_B's src2.
+# → MFMA_B (vgprcd, excluded) reads %acc as AGPR → illegal instruction.
+#
+# MFMA_B is excluded due to HasDstSubregDef (V_MOV subreg write of its dst),
+# NOT HasConflict, so backward propagation does NOT run — no cascade.
+# hasSrc2BridgeConflict for MFMA_B: {AV_MOV×4} all in bb.0 which dominates bb.1
+# → all uses covered → HasConflict=false. HasDstSubregDef=true → excluded.
+#
+# Correct behavior (with fix):
+# hasUseRequiringVGPR treats MFMA_B (excluded) as VGPR-requiring.
+# Src2NeedsVGPR=true → bridge copy for %acc inserted in bb.0.
+# %acc stays VGPR; MFMA_A reads %mapped (AGPR); MFMA_B reads %acc (VGPR).
+#
+# CHECK-LABEL: name: test_partial_exclude_avmov_src2
+# CHECK: bb.0:
+# CHECK: {{.*}}:areg_128_align2 = COPY %acc
+# CHECK: %res_a:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: bb.1:
+# CHECK: %res_b:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca, %srcb, %acc
+name: test_partial_exclude_avmov_src2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; Pressure: 9 × vreg_1024 = 288 ArchVGPRs > gfx90a limit 256 → rewrite fires.
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ %acc:vreg_128_align2 = IMPLICIT_DEF
+ %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ SCHED_BARRIER 0
+
+ %res_a:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ bb.1:
+ %res_b:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %res_b.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %res_b
+ S_ENDPGM 0, implicit %res_a
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
index 71838881aee99..34cf624961f9e 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
@@ -1794,38 +1794,55 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_ADD_U32_e32_]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = COPY [[DEF17]]
; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = COPY [[DEF18]]
; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = COPY [[DEF19]]
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: $scc = IMPLICIT_DEF
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit killed $scc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
- ; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_ADD_U32_e32_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF17]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF18]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[COPY4]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9]]
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11]]
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6]]
+ ; CHECK-NEXT: KILL [[COPY10]], [[COPY5]], [[COPY12]], [[COPY7]], [[COPY14]], [[COPY9]], [[COPY16]], [[COPY11]], [[COPY6]], [[COPY13]], [[COPY8]], [[COPY15]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
@@ -1836,17 +1853,20 @@ body: |
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.6(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_128_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
; CHECK-NEXT: [[DEF20:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[DEF17]], [[DEF18]], [[DEF19]], [[V_ADD_U32_e32_]], [[V_ADD_U32_e32_3]]
+ ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[COPY17]], [[COPY19]], [[COPY18]], [[V_ADD_U32_e32_1]], [[V_ADD_U32_e32_3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -2470,6 +2490,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -2481,27 +2502,34 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: KILL [[COPY2]], [[COPY5]], [[COPY3]], [[COPY6]], [[COPY4]], [[COPY1]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[DS_READ_B128_gfx9_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[COPY7]], 0, 0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[DS_READ_B128_gfx9_]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY7]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -2592,6 +2620,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -2603,27 +2632,34 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: KILL [[COPY2]], [[COPY5]], [[COPY3]], [[COPY6]], [[COPY4]], [[COPY1]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[DS_READ_B128_gfx9_]].sub0, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[COPY7]].sub0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[DS_READ_B128_gfx9_]], [[V_ADD_U32_e32_]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY7]], [[V_ADD_U32_e32_]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -3561,6 +3597,7 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -3570,10 +3607,10 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000)
@@ -3584,21 +3621,27 @@ body: |
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 256, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub0, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub1, 256, 0, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 256, 0, implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub1, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub0, 256, 0, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[DS_READ_B128_gfx9_]], 0, 0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY3]], [[COPY5]], [[COPY4]], [[COPY6]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -5474,23 +5517,24 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DEF16]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF16]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]]
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]].sub0, [[DEF15]], implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[COPY1]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY2]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
>From 68fac5b253a27ac5559cb90763f960ae05ff2eb9 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Wed, 8 Jul 2026 15:51:11 +0000
Subject: [PATCH 2/6] [AMDGPU] Improve src2 bridge-conflict check with joint
bridge coverage
Rework hasSrc2BridgeConflict Check 2 to test path coverage via a backward
CFG walk over the union of non-MAI reaching-def blocks (bridge blocks)
across all candidates sharing the same src2, instead of requiring a single
dominating bridge block. computeExclusionSet precomputes this per-src2
bridge-block union and passes it in.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 133 ++++++++++++--------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 15 ++-
2 files changed, 88 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index e893089412115..b2c97dedbceac 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2452,23 +2452,23 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
return true;
}
-bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
- Register Src2Reg) const {
+bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
+ ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const {
auto &MDT = DAG.LIS->getDomTree();
SmallVector<MachineInstr *, 8> MAIMIs;
- SmallPtrSet<MachineBasicBlock *, 8> BridgeCopyBlocks; // non-MAI def blocks
-
+ SmallVector<MachineInstr *, 8> NonMAIMIs;
for (SlotIndex SI : DefIdxs) {
MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
if (TII->isMFMA(*MI))
MAIMIs.push_back(MI);
else
- BridgeCopyBlocks.insert(MI->getParent());
+ NonMAIMIs.push_back(MI);
}
- if (BridgeCopyBlocks.empty())
- return false; // All defs are MAI; no bridge copies needed.
+ if (NonMAIMIs.empty())
+ return false; // All reaching defs are MAI; no bridge copies needed.
// Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
// The MFMA first writes all lanes (AGPR), then a non-MAI instruction
@@ -2476,13 +2476,6 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
// would read an already-AGPR register and must partially update it —
// a read-modify-write that the bridge-copy mechanism cannot implement.
if (!MAIMIs.empty()) {
- SmallVector<MachineInstr *, 8> NonMAIMIs;
- for (SlotIndex SI : DefIdxs) {
- MachineInstr *MI = DAG.LIS->getInstructionFromIndex(SI);
- if (!TII->isMFMA(*MI))
- NonMAIMIs.push_back(MI);
- }
-
for (MachineInstr *M : MAIMIs)
for (MachineInstr *N : NonMAIMIs)
if (MDT.dominates(M, N))
@@ -2491,45 +2484,60 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
// Early-safe return: if every (MAI, non-MAI) pair is parallel and every
// MAI def is itself a rewrite candidate, the rewrite is safe without
// invoking Check 2.
- if (!NonMAIMIs.empty()) {
- bool AllParallelAndCandidates = true;
- for (MachineInstr *M : MAIMIs) {
- if (!isRewriteCandidate(M)) {
+ bool AllParallelAndCandidates = true;
+ for (MachineInstr *M : MAIMIs) {
+ if (!isRewriteCandidate(M)) {
+ AllParallelAndCandidates = false;
+ break;
+ }
+ for (MachineInstr *N : NonMAIMIs) {
+ if (MDT.dominates(N, M)) {
AllParallelAndCandidates = false;
break;
}
- for (MachineInstr *N : NonMAIMIs) {
- if (MDT.dominates(N, M)) {
- AllParallelAndCandidates = false;
- break;
- }
- }
- if (!AllParallelAndCandidates)
- break;
}
- if (AllParallelAndCandidates)
- return false;
+ if (!AllParallelAndCandidates)
+ break;
}
+ if (AllParallelAndCandidates)
+ return false;
}
- // Check 2: every use of Src2Reg must be dominated by at least one
- // bridge-copy block; otherwise %MappedReg would be undefined on some path.
+ // Check 2: every entry->use path of Src2Reg must cross a bridge block (where
+ // rewrite() inserts the bridge copy); otherwise %MappedReg is undefined on
+ // the bypassing path. BridgeBlocks (precomputed in computeExclusionSet) is
+ // the union of non-MAI *reaching-def* blocks of Src2Reg across all candidates
+ // sharing it -- see computeExclusionSet for why the whole-register union and
+ // reaching-def (not all-def) blocks are both required.
//
- // use_nodbg_operands is used here (not findReachingUses) because rewrite()
- // replaces the src2 operand of the MFMA with %MappedReg uniformly — it does
- // not distinguish which reaching def flows to which use. %MappedReg is
- // defined only in bridge-copy blocks (after non-MAI defs); if any use of
- // Src2Reg is in a block not dominated by any bridge-copy block, %MappedReg
- // would be undefined on that path regardless of whether the use is reached
- // by a MAI or non-MAI def. findReachingUses(non-MAI RD) would miss uses
- // that arrive only via MAI defs or other defs (e.g. IMPLICIT_DEF on a
- // bypass path), causing a false negative and silent undefined-read.
+ // Bridge blocks must *jointly* cover every path: two blocks may each cover a
+ // disjoint set of paths with neither dominating the use, so a plain dominance
+ // test is too strict. A backward walk from the use block that stops at
+ // bridge blocks decides this -- reaching entry means an uncovered path
+ // exists. All uses (use_nodbg_operands, not just src2 consumers) must be
+ // covered: a plain COPY of Src2Reg also reads %MappedReg after rewrite.
+ auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
+ if (BridgeBlocks.contains(UseBlock))
+ return true;
+ SmallPtrSet<const MachineBasicBlock *, 16> Visited;
+ SmallVector<const MachineBasicBlock *, 16> Worklist(UseBlock->pred_begin(),
+ UseBlock->pred_end());
+ while (!Worklist.empty()) {
+ const MachineBasicBlock *B = Worklist.pop_back_val();
+ if (BridgeBlocks.contains(B))
+ continue; // This path is covered; do not walk past the bridge block.
+ if (B->pred_empty())
+ return false; // Reached entry without crossing a bridge block.
+ if (!Visited.insert(B).second)
+ continue;
+ Worklist.append(B->pred_begin(), B->pred_end());
+ }
+ return true;
+ };
+
for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
- bool Covered = any_of(BridgeCopyBlocks, [&](const MachineBasicBlock *B) {
- return MDT.dominates(B, UseBlock);
- });
- if (!Covered)
+ if (!CoveredByBridgeSet(UseBlock))
return true;
}
@@ -2627,6 +2635,31 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
// Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
// and backward (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
+
+ // Precompute, per shared src2 register, the union of its non-MAI reaching-def
+ // blocks across all candidates -- exactly where rewrite() inserts bridge
+ // copies. Check 2 needs the whole-register union, not one candidate's
+ // reaching defs, since a sibling candidate's def in another block also
+ // supplies a value to a shared use. Reaching-def (not all-def) blocks are
+ // used so defs killed on a bypass path and entry IMPLICIT_DEFs stay excluded,
+ // keeping a genuine bypass reported as a conflict.
+ DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
+ Src2BridgeBlocks;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2 || !Src2->isReg())
+ continue;
+ Register Src2Reg = Src2->getReg();
+ auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ SmallVector<SlotIndex, 8> Src2Defs;
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
+ if (RD && !TII->isMFMA(*RD))
+ Blocks.insert(RD->getParent());
+ }
+ }
+
for (MachineInstr *MI : RewriteSet) {
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
Register DstReg = MI->getOperand(0).getReg();
@@ -2635,18 +2668,8 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
SmallVector<SlotIndex, 8> Src2Defs;
if (Src2->isReg()) {
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- LLVM_DEBUG({
- dbgs() << "[computeExclusionSet] candidate: " << *MI;
- dbgs() << " src2 reaching defs (" << Src2Defs.size() << "):\n";
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *D = DAG.LIS->getInstructionFromIndex(SI);
- dbgs() << " " << SI << " opcode=" << (D ? (int)D->getOpcode() : -1)
- << "\n";
- if (D)
- dbgs() << " " << *D;
- }
- });
- HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg());
+ HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg(),
+ Src2BridgeBlocks[Src2->getReg()]);
}
bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 25a696f3e540b..900b50c9a716d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -491,11 +491,16 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Early-safe return: if all (MAI, non-MAI) pairs are parallel in the CFG
/// and every MAI def is itself a candidate, the non-MAI defs can be bridged
/// without conflict; Check 2 is skipped.
- /// Check 2: every use of \p Src2Reg must be dominated by at least one
- /// non-MAI def block (where a bridge copy is inserted); otherwise %MappedReg
- /// is undefined on some CFG path to a use.
- bool hasSrc2BridgeConflict(ArrayRef<SlotIndex> DefIdxs,
- Register Src2Reg) const;
+ /// Check 2: on every path from the entry to a use of \p Src2Reg, some bridge
+ /// block in \p BridgeBlocks (a non-MAI def block of Src2Reg, where a bridge
+ /// copy is inserted) must be crossed; otherwise %MappedReg is undefined on
+ /// that path. \p BridgeBlocks is precomputed in computeExclusionSet as the
+ /// union, over all candidates sharing Src2Reg, of the non-MAI reaching-def
+ /// blocks of their src2 — this is what rewrite() actually inserts copies
+ /// after, so a single candidate's own reaching defs are not enough.
+ bool hasSrc2BridgeConflict(
+ ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const;
/// Returns true if reclassifying \p DstReg to AGPR would require bridge
/// copies beyond rewrite()'s capability (full-register copies only).
>From d691e11217b467dd2fa6828c8f011c9d7e8712f6 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 04:28:53 +0000
Subject: [PATCH 3/6] [AMDGPU] Add src2 connectivity check to MFMA form rewrite
guard
Extend hasSrc2BridgeConflict with a connectivity check: if src2 has >=2
mutually non-dominating non-AGPR-form partial-subreg reaching-def blocks,
redirecting uses to %MappedReg would fragment the original live interval.
Runs independently of the early-safe path.
Add regression tests test_src2_shared_diamond_joint_coverage and
test_earlysafe_connectivity_fragment.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 110 +++++++----
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 8 +-
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 187 ++++++++++++++++++
3 files changed, 270 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b2c97dedbceac..8f30ab515cdc3 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2454,7 +2454,9 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
- const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const {
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
+ const {
auto &MDT = DAG.LIS->getDomTree();
SmallVector<MachineInstr *, 8> MAIMIs;
@@ -2475,15 +2477,19 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
// partially overwrites some lanes. A bridge copy at the non-MAI def
// would read an already-AGPR register and must partially update it —
// a read-modify-write that the bridge-copy mechanism cannot implement.
+ bool SkipCheck2 = false;
if (!MAIMIs.empty()) {
for (MachineInstr *M : MAIMIs)
for (MachineInstr *N : NonMAIMIs)
if (MDT.dominates(M, N))
return true;
- // Early-safe return: if every (MAI, non-MAI) pair is parallel and every
- // MAI def is itself a rewrite candidate, the rewrite is safe without
- // invoking Check 2.
+ // Early-safe: if every MAI reaching def is a candidate and parallel to
+ // every non-MAI def (Check 1 ruled out MAI dominating non-MAI; here the
+ // reverse), then %MappedReg is defined on every path to a use -- a
+ // candidate MAI gives an AGPR value directly, a non-MAI def carries a
+ // bridge copy -- so Check 2 can be skipped. Check 3 is independent and
+ // must still run.
bool AllParallelAndCandidates = true;
for (MachineInstr *M : MAIMIs) {
if (!isRewriteCandidate(M)) {
@@ -2500,22 +2506,17 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
break;
}
if (AllParallelAndCandidates)
- return false;
+ SkipCheck2 = true;
}
- // Check 2: every entry->use path of Src2Reg must cross a bridge block (where
- // rewrite() inserts the bridge copy); otherwise %MappedReg is undefined on
- // the bypassing path. BridgeBlocks (precomputed in computeExclusionSet) is
- // the union of non-MAI *reaching-def* blocks of Src2Reg across all candidates
- // sharing it -- see computeExclusionSet for why the whole-register union and
- // reaching-def (not all-def) blocks are both required.
- //
- // Bridge blocks must *jointly* cover every path: two blocks may each cover a
- // disjoint set of paths with neither dominating the use, so a plain dominance
- // test is too strict. A backward walk from the use block that stops at
- // bridge blocks decides this -- reaching entry means an uncovered path
- // exists. All uses (use_nodbg_operands, not just src2 consumers) must be
- // covered: a plain COPY of Src2Reg also reads %MappedReg after rewrite.
+ // Check 2: every entry->use path of Src2Reg must hit a bridge block (a
+ // non-MAI def block where rewrite() inserts the bridge copy; the use block
+ // itself counts), else %MappedReg is undefined on the bypassing path. Bridge
+ // blocks must *jointly* cover every path -- one may not dominate the use, so
+ // a dominance test is too strict; a backward walk stopping at bridge blocks
+ // that reaches entry means an uncovered path exists. Every use counts (a
+ // plain COPY of Src2Reg also reads %MappedReg). BridgeBlocks: see
+ // computeExclusionSet.
auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
if (BridgeBlocks.contains(UseBlock))
return true;
@@ -2535,9 +2536,36 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
return true;
};
- for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
- const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
- if (!CoveredByBridgeSet(UseBlock))
+ if (!SkipCheck2) {
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
+ if (!CoveredByBridgeSet(UseBlock))
+ return true;
+ }
+ }
+
+ // Check 3: original-register connectivity. rewrite() redirects Src2Reg's
+ // uses to %MappedReg, so if non-AGPR-form partial subreg defs sit in >=2
+ // blocks with none dominating all uses, Src2Reg's live interval splits into
+ // disconnected fragments. AGPR-form defs are excluded from
+ // RedefPartialBlocks (skipped by Case1), preserving the diamond
+ // joint-coverage case.
+ if (RedefPartialBlocks.size() >= 2) {
+ bool SingleDominator = false;
+ for (const MachineBasicBlock *B : RedefPartialBlocks) {
+ bool DomAllUses = true;
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
+ if (!MDT.dominates(B, UseMO.getParent()->getParent())) {
+ DomAllUses = false;
+ break;
+ }
+ }
+ if (DomAllUses) {
+ SingleDominator = true;
+ break;
+ }
+ }
+ if (!SingleDominator)
return true;
}
@@ -2627,8 +2655,9 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
// Per-MI checks run cheapest-first:
// 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
- // a read-modify-write on AGPR (MAI def dominates non-MAI def), or
- // absent on some CFG path to a src2 use.
+ // a read-modify-write on AGPR (MAI def dominates non-MAI def), the
+ // bridge value is absent on some CFG path to a src2 use, or the
+ // redirected original register splits into disconnected fragments.
// 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
// be reclassified to AGPR. Skipped when check 1 already forces
// exclusion (!HasConflict &&).
@@ -2636,27 +2665,39 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
// and backward (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
- // Precompute, per shared src2 register, the union of its non-MAI reaching-def
- // blocks across all candidates -- exactly where rewrite() inserts bridge
- // copies. Check 2 needs the whole-register union, not one candidate's
- // reaching defs, since a sibling candidate's def in another block also
- // supplies a value to a shared use. Reaching-def (not all-def) blocks are
- // used so defs killed on a bypass path and entry IMPLICIT_DEFs stay excluded,
- // keeping a genuine bypass reported as a conflict.
+ // Per src2 register, union reaching-def blocks across all candidates (one
+ // candidate sees only one path): non-MFMA -> Src2BridgeBlocks (Check 2), plus
+ // non-AGPR-form partial-subreg -> Src2RedefPartialBlocks (Check 3). Reaching-
+ // def (not all-def) keeps bypass-killed and entry IMPLICIT_DEFs out.
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2BridgeBlocks;
+ DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
+ Src2RedefPartialBlocks;
+ DenseSet<Register> CandSrc2Regs;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
for (MachineInstr *MI : RewriteSet) {
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
if (!Src2 || !Src2->isReg())
continue;
Register Src2Reg = Src2->getReg();
auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
SmallVector<SlotIndex, 8> Src2Defs;
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
for (SlotIndex SI : Src2Defs) {
MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
- if (RD && !TII->isMFMA(*RD))
- Blocks.insert(RD->getParent());
+ if (!RD || TII->isMFMA(*RD))
+ continue;
+ Blocks.insert(RD->getParent());
+ // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
+ // (AGPR-form skipped by Case1; full-width defs stay connected).
+ if (!isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII) &&
+ getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
+ RedefBlocks.insert(RD->getParent());
}
}
@@ -2668,8 +2709,9 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
SmallVector<SlotIndex, 8> Src2Defs;
if (Src2->isReg()) {
findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- HasConflict = hasSrc2BridgeConflict(Src2Defs, Src2->getReg(),
- Src2BridgeBlocks[Src2->getReg()]);
+ HasConflict = hasSrc2BridgeConflict(
+ Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
+ Src2RedefPartialBlocks[Src2->getReg()]);
}
bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 900b50c9a716d..5909ba4d28c8c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -498,9 +498,15 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// union, over all candidates sharing Src2Reg, of the non-MAI reaching-def
/// blocks of their src2 — this is what rewrite() actually inserts copies
/// after, so a single candidate's own reaching defs are not enough.
+ /// Check 3: if \p RedefPartialBlocks (non-AGPR-form partial subreg
+ /// reaching-def blocks of Src2Reg) holds >=2 mutually non-dominating blocks
+ /// with no single block dominating all uses, redirecting uses to %MappedReg
+ /// splits the original Src2Reg live interval into disconnected components.
bool hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
- const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks) const;
+ const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
+ const;
/// Returns true if reclassifying \p DstReg to AGPR would require bridge
/// copies beyond rewrite()'s capability (full-register copies only).
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
index 58a1ca4c8dd04..2d81741ec9759 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -40,6 +40,14 @@
entry:
unreachable
}
+ define void @test_src2_shared_diamond_joint_coverage() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_earlysafe_connectivity_fragment() #0 {
+ entry:
+ unreachable
+ }
attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
...
@@ -633,3 +641,182 @@ body: |
KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8, %res_b
S_ENDPGM 0, implicit %res_a
+...
+
+---
+#
+# Shared-src2 diamond where the bridge blocks JOINTLY cover the merge use, but
+# no single block dominates it. This is the opt_bug shape and it exercises the
+# whole-register bridge-set union in computeExclusionSet / hasSrc2BridgeConflict.
+#
+# CFG (diamond):
+#
+# bb.0
+# / \
+# bb.1 bb.2 ← bb.1: %acc.subN = AV_MOV 0 (non-MAI init, left branch)
+# \ / bb.2: %acc.subN = AV_MOV 0 (non-MAI init, right branch)
+# bb.3 ← %res = MFMA(a, b, %acc) (src2 = %acc)
+# %res2 = MFMA(a, b, %acc) (src2 = %acc, second consumer)
+#
+# %acc has two non-MAI def blocks: bb.1 (left) and bb.2 (right). Neither
+# dominates bb.3, but together they cover every entry->bb.3 path. The old
+# per-candidate check built BridgeCopyBlocks from a single candidate's reaching
+# defs and wrongly reported "uncovered". The union across both candidates that
+# share %acc as src2 covers bb.3 → no conflict → both MFMAs rewrite to AGPR.
+#
+# CHECK-LABEL: name: test_src2_shared_diamond_joint_coverage
+# CHECK: bb.3:
+# CHECK: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK: %res2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+# CHECK-NOT: vgprcd
+name: test_src2_shared_diamond_joint_coverage
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 x vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ ; Left branch: full-width non-MAI init of %acc.
+ bb.1:
+ successors: %bb.3(0x80000000)
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; Right branch: full-width non-MAI init of %acc.
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ undef %acc.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub2:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %acc.sub3:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ S_BRANCH %bb.3
+
+ ; Merge: two MFMAs share %acc as src2. Each is reached by only one branch's
+ ; init in its own reaching-def set; only the cross-candidate union covers bb.3.
+ bb.3:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ S_ENDPGM 0
+...
+
+---
+#
+# The early-safe path (skipping the bridge dominance/coverage checks) must NOT
+# suppress the connectivity check. bb.5's MAI def is parallel to every non-MAI
+# def, so hasSrc2BridgeConflict takes the early-safe path and skips bridge
+# coverage -- but connectivity still runs: the two agnostic partial-subreg COPY
+# defs in bb.2/bb.3, neither dominating the bb.6 use, fragment %acc's live range
+# -> conflict -> both MFMAs stay in vgprcd form.
+#
+# CFG:
+#
+# bb.0
+# / \
+# bb.1 bb.5 ← bb.5: %acc = MFMA (MAI full-reg def, candidate)
+# / \ \
+# bb.2 bb.3 \ ← bb.2/bb.3: %acc.subN = COPY (agnostic partial defs)
+# \ / \
+# bb.4 \
+# \ /
+# \ /
+# bb.6 ← %res = MFMA(a, b, %acc) (src2 = %acc)
+#
+# CHECK-LABEL: name: test_earlysafe_connectivity_fragment
+# CHECK: bb.5:
+# CHECK: %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: bb.6:
+# CHECK: %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: V_MFMA_F32_16X16X4F32_e64
+name: test_earlysafe_connectivity_fragment
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ ; 9 x vreg_1024 = 288 ArchVGPRs pressure (> gfx90a limit 256).
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %accinit:vreg_128_align2 = IMPLICIT_DEF
+ %cpsrc0:vgpr_32 = IMPLICIT_DEF
+ %cpsrc1:vgpr_32 = IMPLICIT_DEF
+ %cpsrc2:vgpr_32 = IMPLICIT_DEF
+ %cpsrc3:vgpr_32 = IMPLICIT_DEF
+
+ S_CBRANCH_SCC1 %bb.5, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ ; branch A: agnostic partial-subreg def (fragments live range, not a subreg-def conflict).
+ bb.2:
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
+
+ ; branch B: agnostic partial-subreg def (fragments live range).
+ bb.3:
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.6(0x80000000)
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %accinit:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.6
+
+ bb.6:
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
>From 5461be0d51205b2de007a02436f1afaec14f5f87 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 07:11:19 +0000
Subject: [PATCH 4/6] [AMDGPU] Iterate MFMA-form exclusion to a fixpoint
computeExclusionSet classified src2 reaching defs in a single pass.
Excluding one candidate makes its dst/src2 no longer AGPR-form, which can
expose a conflict in another candidate the single pass never revisits --
letting an unsafe rewrite through and splitting a live interval. Instead,
recompute the block maps from the still-live candidates and iterate until
a round adds nothing; exclusions only grow over a finite set, so it
terminates.
Add test_src2_exclusion_fixpoint covering the cascade where A forces B.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 199 ++++++++++--------
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 97 +++++++++
2 files changed, 206 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 8f30ab515cdc3..df44f4c525bf3 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2472,11 +2472,9 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
if (NonMAIMIs.empty())
return false; // All reaching defs are MAI; no bridge copies needed.
- // Check 1: MAI def dominates non-MAI def (partial subreg overwrite).
- // The MFMA first writes all lanes (AGPR), then a non-MAI instruction
- // partially overwrites some lanes. A bridge copy at the non-MAI def
- // would read an already-AGPR register and must partially update it —
- // a read-modify-write that the bridge-copy mechanism cannot implement.
+ // Check 1: MAI def dominates non-MAI def. The MFMA writes all lanes (AGPR),
+ // then a non-MAI def partially overwrites some; a bridge copy there would be
+ // a read-modify-write on AGPR, which the bridge mechanism cannot implement.
bool SkipCheck2 = false;
if (!MAIMIs.empty()) {
for (MachineInstr *M : MAIMIs)
@@ -2510,12 +2508,11 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
}
// Check 2: every entry->use path of Src2Reg must hit a bridge block (a
- // non-MAI def block where rewrite() inserts the bridge copy; the use block
- // itself counts), else %MappedReg is undefined on the bypassing path. Bridge
- // blocks must *jointly* cover every path -- one may not dominate the use, so
- // a dominance test is too strict; a backward walk stopping at bridge blocks
- // that reaches entry means an uncovered path exists. Every use counts (a
- // plain COPY of Src2Reg also reads %MappedReg). BridgeBlocks: see
+ // non-MAI def block where rewrite() inserts the bridge copy, or the use block
+ // itself), else %MappedReg is undefined on the bypassing path. Bridge blocks
+ // must *jointly* cover every path (dominance is too strict), so a backward
+ // walk that reaches entry without crossing one proves an uncovered path.
+ // Every use counts, incl. a plain COPY. BridgeBlocks: see
// computeExclusionSet.
auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
if (BridgeBlocks.contains(UseBlock))
@@ -2545,11 +2542,10 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
}
// Check 3: original-register connectivity. rewrite() redirects Src2Reg's
- // uses to %MappedReg, so if non-AGPR-form partial subreg defs sit in >=2
- // blocks with none dominating all uses, Src2Reg's live interval splits into
- // disconnected fragments. AGPR-form defs are excluded from
- // RedefPartialBlocks (skipped by Case1), preserving the diamond
- // joint-coverage case.
+ // uses to %MappedReg, so non-AGPR-form partial subreg defs in >=2 blocks with
+ // none dominating all uses split Src2Reg's live interval into disconnected
+ // fragments. AGPR-form defs are absent from RedefPartialBlocks (skipped by
+ // Case1), preserving the diamond joint-coverage case.
if (RedefPartialBlocks.size() >= 2) {
bool SingleDominator = false;
for (const MachineBasicBlock *B : RedefPartialBlocks) {
@@ -2599,12 +2595,9 @@ void RewriteMFMAFormStage::propagateExclusionForward(
}
}
-// rewrite()'s design principle: reclassify DstReg from VGPR to AGPR class so
-// that the MFMA emits its result directly into AGPR, eliminating the need for
-// a post-MFMA VGPR→AGPR copy. For this reclassification to be legal, every
-// def and every use of DstReg throughout its live range must support
-// AGPR-class operands. hasDstSubregConflict checks this before rewriting:
-//
+// rewrite() reclassifies DstReg to AGPR so the MFMA writes its result directly
+// into AGPR (no post-MFMA copy). That is legal only if every def and use of
+// DstReg across its live range supports AGPR-class operands; check that here.
bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
MachineInstr *MFMA) {
SmallVector<MachineOperand *, 8> DstReachingUses;
@@ -2653,16 +2646,10 @@ bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
- // Per-MI checks run cheapest-first:
- // 1. hasSrc2BridgeConflict: bridge COPY after non-MAI src2 def would be
- // a read-modify-write on AGPR (MAI def dominates non-MAI def), the
- // bridge value is absent on some CFG path to a src2 use, or the
- // redirected original register splits into disconnected fragments.
- // 2. hasDstSubregConflict: DstReg has non-MAI subreg writers that cannot
- // be reclassified to AGPR. Skipped when check 1 already forces
- // exclusion (!HasConflict &&).
- // Exclusion propagates forward (dst→src2 chain via propagateExclusionForward)
- // and backward (MAI reaching-defs of a conflicted src2).
+ // Per-MI checks, cheapest-first: (1) hasSrc2BridgeConflict, then
+ // (2) hasDstSubregConflict (skipped when 1 already excludes). Exclusion
+ // propagates forward (dst->src2 via propagateExclusionForward) and backward
+ // (MAI reaching-defs of a conflicted src2).
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
// Per src2 register, union reaching-def blocks across all candidates (one
@@ -2673,75 +2660,107 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
Src2BridgeBlocks;
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2RedefPartialBlocks;
- DenseSet<Register> CandSrc2Regs;
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (!Src2 || !Src2->isReg())
- continue;
- Register Src2Reg = Src2->getReg();
- auto &Blocks = Src2BridgeBlocks[Src2Reg];
- auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
- SmallVector<SlotIndex, 8> Src2Defs;
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
- if (!RD || TII->isMFMA(*RD))
+
+ // Rebuild the block maps from the still-live candidates \p Active: excluding
+ // a candidate makes its src2/MAI def non-AGPR-form (isReachingDefAGPRForm
+ // reads Active), which can add new Check 3 redef sources -- so the full
+ // pre-exclusion set would miss them.
+ auto recomputeBlocks = [&](const SmallSetVector<MachineInstr *, 16> &Active) {
+ Src2BridgeBlocks.clear();
+ Src2RedefPartialBlocks.clear();
+ DenseSet<Register> CandSrc2Regs;
+ for (MachineInstr *MI : Active) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
+ for (MachineInstr *MI : Active) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2 || !Src2->isReg())
continue;
- Blocks.insert(RD->getParent());
- // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
- // (AGPR-form skipped by Case1; full-width defs stay connected).
- if (!isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII) &&
- getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
- RedefBlocks.insert(RD->getParent());
+ Register Src2Reg = Src2->getReg();
+ auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
+ SmallVector<SlotIndex, 8> Src2Defs;
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
+ if (!RD || TII->isMFMA(*RD))
+ continue;
+ Blocks.insert(RD->getParent());
+ // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
+ // (AGPR-form skipped by Case1; full-width defs stay connected).
+ if (!isReachingDefAGPRForm(RD, Active, CandSrc2Regs, *TII) &&
+ getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
+ RedefBlocks.insert(RD->getParent());
+ }
}
- }
+ };
- for (MachineInstr *MI : RewriteSet) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- Register DstReg = MI->getOperand(0).getReg();
+ // Fixpoint: each exclusion can expose new conflicts in others (see
+ // recomputeBlocks). Iterate to convergence; exclusions only grow over a
+ // finite candidate set, so this terminates.
+ bool Changed;
+ do {
+ Changed = false;
- bool HasConflict = false;
- SmallVector<SlotIndex, 8> Src2Defs;
- if (Src2->isReg()) {
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- HasConflict = hasSrc2BridgeConflict(
- Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
- Src2RedefPartialBlocks[Src2->getReg()]);
- }
- bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+ SmallSetVector<MachineInstr *, 16> Active;
+ for (MachineInstr *MI : RewriteSet)
+ if (!ExcludedMFMAs.count(MI))
+ Active.insert(MI);
- if (!HasConflict && !HasDstSubregDef)
- continue;
+ recomputeBlocks(Active);
- if (ExcludedMFMAs.insert(MI).second) {
- LLVM_DEBUG(
- dbgs() << "[computeExclusionSet] exclude MFMA ("
- << (HasConflict ? "src2 dominance conflict" : "")
- << (HasConflict && HasDstSubregDef ? " + " : "")
- << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
- << "): " << *MI);
- propagateExclusionForward(MI, ExcludedMFMAs);
- }
+ for (MachineInstr *MI : Active) {
+ // May have been excluded by forward/backward propagation earlier this
+ // round.
+ if (ExcludedMFMAs.count(MI))
+ continue;
+
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ Register DstReg = MI->getOperand(0).getReg();
+
+ bool HasConflict = false;
+ SmallVector<SlotIndex, 8> Src2Defs;
+ if (Src2->isReg()) {
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ HasConflict = hasSrc2BridgeConflict(
+ Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
+ Src2RedefPartialBlocks[Src2->getReg()]);
+ }
+ bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+
+ if (!HasConflict && !HasDstSubregDef)
+ continue;
+
+ if (ExcludedMFMAs.insert(MI).second) {
+ LLVM_DEBUG(
+ dbgs() << "[computeExclusionSet] exclude MFMA ("
+ << (HasConflict ? "src2 dominance conflict" : "")
+ << (HasConflict && HasDstSubregDef ? " + " : "")
+ << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
+ << "): " << *MI);
+ propagateExclusionForward(MI, ExcludedMFMAs);
+ Changed = true;
+ }
- // Backward: exclude MAI reaching-defs that are themselves candidates.
- if (HasConflict) {
+ // Backward: exclude MAI reaching-defs that are themselves candidates.
+ if (!HasConflict)
+ continue;
for (SlotIndex SI : Src2Defs) {
MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
- if (TII->isMFMA(*DefMI) && isRewriteCandidate(DefMI) &&
- ExcludedMFMAs.insert(DefMI).second) {
- LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
- "(backward from src2 dominance conflict): "
- << *DefMI);
- propagateExclusionForward(DefMI, ExcludedMFMAs);
- }
+ if (!TII->isMFMA(*DefMI) || !isRewriteCandidate(DefMI) ||
+ !ExcludedMFMAs.insert(DefMI).second)
+ continue;
+ LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
+ "(backward from src2 dominance conflict): "
+ << *DefMI);
+ propagateExclusionForward(DefMI, ExcludedMFMAs);
+ Changed = true;
}
}
- }
+ } while (Changed);
+
return ExcludedMFMAs;
}
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
index 2d81741ec9759..7841edcf8671c 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -48,6 +48,10 @@
entry:
unreachable
}
+ define void @test_src2_exclusion_fixpoint() #0 {
+ entry:
+ unreachable
+ }
attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
...
@@ -820,3 +824,96 @@ body: |
KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
S_ENDPGM 0
...
+
+---
+#
+# Exclusion is a fixpoint: excluding one candidate re-classifies another's src2.
+#
+# bb.0
+# / \
+# bb.1 bb.2 %rA.subN = V_MOV (non-AGPR-form partial def, fragments %rA)
+# \ /
+# bb.3 %resA = MFMA(.., %rA) candidate A, src2=%rA
+# / \
+# bb.4 bb.5 %acc.subN = COPY %rA.subN (partial subreg def; source = A's src2)
+# \ /
+# bb.6 %resB = MFMA(.., %acc) candidate B, src2=%acc
+#
+# A is excluded (non-dominating partial defs fragment %rA), so %rA stays VGPR and
+# the COPYs feeding %acc lose AGPR-form -- B must be excluded too. A single pass
+# misses B and rewrites it, splitting %acc's live interval; the fixpoint
+# re-classification catches B, keeping both MFMAs in vgprcd (VGPR) form.
+#
+# CHECK-LABEL: name: test_src2_exclusion_fixpoint
+# CHECK: %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_src2_exclusion_fixpoint
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %cond:sgpr_32 = COPY $sgpr6
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %rA:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.4:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.6:
+ %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
>From 3dc2a7a69de85a22f8442c7a839590e9d3d12509 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 14 Jul 2026 10:50:03 +0000
Subject: [PATCH 5/6] [AMDGPU] Factor MFMA-form copy analysis into shared
copy-plan analysis
initHeuristics and rewrite duplicated the logic for which src2/dst copies a
rewrite needs. Extract it into an analyzeCopyPoints helper returning an
MFMACopyPlan, so both compute the same plan from the same code and the cost
model and the rewrite agree by construction. Also switch to a post-exclusion
RewriteSet so downstream queries test membership instead of carrying a
separate ExcludedMFMAs set.
analyzeCopyPoints is called once per site (not cached) so each plan reflects
the current MIR, including edits from earlier candidates sharing a dst vreg.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 236 +++++++++-----------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 30 ++-
2 files changed, 136 insertions(+), 130 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index df44f4c525bf3..313671f83c655 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2393,18 +2393,13 @@ static bool isReachingDefAGPRForm(
bool RewriteMFMAFormStage::hasUseRequiringVGPR(
ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallSetVector<MachineInstr *, 16> &RewriteSet,
- const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs) {
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
for (SlotIndex RDIdx : Src2ReachingDefs) {
const MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- // If RD is a non-excluded MFMA candidate, its dst will be reclassified to
- // AGPR and Case 2 will insert vreg bridge copies for all non-MAI uses of
- // its dst. Those uses therefore do not impose a VGPR constraint on src2.
- // The !ExcludedMFMAs.count(RD) guard is defensive:
- // propagateExclusionForward ensures that if an excluded MFMA's dst is used
- // as src2, the consumer MFMA is also excluded and hasUseRequiringVGPR is
- // never called for it.
- if (TII->isMAI(*RD) && RewriteSet.contains(RD) && !ExcludedMFMAs.count(RD))
+ // A rewritable candidate's dst becomes AGPR and Case 2 bridges its non-MAI
+ // uses, so it imposes no VGPR constraint on src2. Excluded MFMAs are no
+ // longer in RewriteSet.
+ if (TII->isMAI(*RD) && RewriteSet.contains(RD))
continue;
SmallVector<MachineOperand *, 8> ReachingUses;
findReachingUses(RD, DAG.LIS, ReachingUses);
@@ -2412,8 +2407,7 @@ bool RewriteMFMAFormStage::hasUseRequiringVGPR(
const MachineInstr *UseMI = UseMO->getParent();
if (UseMI->isCopy())
continue;
- if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI) &&
- !ExcludedMFMAs.count(UseMI))
+ if (TII->isMAI(*UseMI) && RewriteSet.contains(UseMI))
continue;
return true;
}
@@ -2764,39 +2758,91 @@ SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
return ExcludedMFMAs;
}
+RewriteMFMAFormStage::MFMACopyPlan RewriteMFMAFormStage::analyzeCopyPoints(
+ MachineInstr *MI, const SmallSetVector<MachineInstr *, 16> &GroupSet,
+ const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR) {
+ MFMACopyPlan Plan;
+
+ // Case 1: the reaching defs of the src2 operand that need a bridge copy.
+ // If src2 stays VGPR, every reaching def needs a copy; otherwise reaching
+ // defs already in AGPR form (group members, av-movs, group copies) don't.
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2->isReg()) {
+ SmallVector<SlotIndex, 8> Src2ReachingDefs;
+ findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
+ for (SlotIndex RDIdx : Src2ReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
+ if (!Src2NeedsVGPR && isReachingDefAGPRForm(RD, GroupSet, Src2Regs, *TII))
+ continue;
+ Plan.Src2DefsNeedingCopy.insert(RD);
+ }
+ }
+
+ // Case 2 and Case 3: the reaching uses of the dst, and the non-MAI reaching
+ // defs of those reaching uses. Group members read the AGPR result directly
+ // and need no copy.
+ SmallVector<MachineOperand *, 8> DstReachingUses;
+ findReachingUses(MI, DAG.LIS, DstReachingUses);
+ for (MachineOperand *RUOp : DstReachingUses) {
+ MachineInstr *UserMI = RUOp->getParent();
+ if (TII->isMAI(*UserMI) && GroupSet.contains(UserMI))
+ continue;
+
+ Plan.DstUsesNeedingCopy.insert(RUOp);
+
+ // Non-rewritten MAI: its defs aren't being reclassified.
+ if (TII->isMAI(*UserMI))
+ continue;
+
+ SmallVector<SlotIndex, 8> DstUsesReachingDefs;
+ findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
+ for (SlotIndex RDIndex : DstUsesReachingDefs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
+ if (TII->isMAI(*RD))
+ continue;
+ Plan.DstUseDefsNeedingCopy.insert(RD);
+ }
+ }
+
+ return Plan;
+}
+
bool RewriteMFMAFormStage::initHeuristics(
std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
bool Changed = false;
- // Pass 1: collect candidate group.
- // RewriteSet/CandSrc2Regs are needed by isReachingDefAGPRForm and
- // hasUseRequiringVGPR; collect them before any setDesc/setRegClass changes.
+ // Pass 1: collect the candidate group (pre-exclusion) before any setDesc/
+ // setRegClass changes; computeExclusionSet needs the complete group.
SmallSetVector<MachineInstr *, 16> RewriteSet;
- DenseSet<Register> CandSrc2Regs;
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
- continue;
- RewriteSet.insert(&MI);
- MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
- }
-
- // Phase 1: identify candidates that cannot be safely rewritten.
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (isRewriteCandidate(&MI))
+ RewriteSet.insert(&MI);
+
+ // Remove unsafe candidates so RewriteSet becomes the post-exclusion group --
+ // both the Pass 2 worklist and the "will be AGPR-form" set queried
+ // downstream.
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs =
computeExclusionSet(RewriteSet);
+ for (MachineInstr *Excluded : ExcludedMFMAs) {
+ LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *Excluded);
+ RewriteSet.remove(Excluded);
+ }
- // Pass 2: compute heuristics for non-excluded candidates.
+ // src2 regs of the post-exclusion group only: an excluded candidate's src2
+ // stays VGPR, so isReachingDefAGPRForm must not treat a COPY from it as
+ // AGPR-form.
+ DenseSet<Register> CandSrc2Regs;
for (MachineInstr *MI : RewriteSet) {
- if (ExcludedMFMAs.count(MI)) {
- LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *MI);
- continue;
- }
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
+ // Pass 2: compute heuristics for the remaining (rewritable) candidates.
+ for (MachineInstr *MI : RewriteSet) {
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
assert(ReplacementOp != -1);
@@ -2804,21 +2850,22 @@ bool RewriteMFMAFormStage::initHeuristics(
MI->setDesc(TII->get(ReplacementOp));
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+
+ // Cache whether src2 must stay VGPR, then get the copy plan reused by
+ // rewrite().
+ bool Src2NeedsVGPR = false;
if (Src2->isReg()) {
SmallVector<SlotIndex, 8> Src2ReachingDefs;
findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
+ Src2NeedsVGPR = hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet);
+ }
+ Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR =
- hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet, ExcludedMFMAs);
- Src2NeedsVGPRCache[MI] = Src2NeedsVGPR;
+ MFMACopyPlan Plan =
+ analyzeCopyPoints(MI, RewriteSet, CandSrc2Regs, Src2NeedsVGPR);
- for (SlotIndex RDIdx : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIdx);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII))
- continue;
+ if (Src2->isReg()) {
+ for (MachineInstr *RD : Plan.Src2DefsNeedingCopy) {
// If this reaching def writes into the src2 register itself (possibly
// a partial subreg write), the def will be reclassified together with
// src2 and does not need a bridge copy.
@@ -2831,39 +2878,13 @@ bool RewriteMFMAFormStage::initHeuristics(
}
MachineOperand &Dst = MI->getOperand(0);
- SmallVector<MachineOperand *, 8> DstReachingUses;
-
- findReachingUses(MI, DAG.LIS, DstReachingUses);
-
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteSet.contains(UserMI))
- continue;
-
- // For any user of the result of the MFMA which is not an MFMA, we
- // insert a copy. For a given register, we will only insert one copy
- // per user block.
- CopyForUse[UserMI->getParent()].insert(RUOp->getReg());
-
- if (TII->isMAI(*UserMI))
- continue;
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
-
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
-
- // For any definition of the user of the MFMA which is not an MFMA,
- // we insert a copy. We do this to transform all the reaching defs
- // of this use to AGPR. By doing this, we can insert a copy from
- // AGPR to VGPR at the user rather than after the MFMA.
- CopyForDef.insert(RD);
- }
- }
+ // Case 2: one copy per (user block, register). Case 3: one copy per
+ // non-MAI reaching def of the dst uses.
+ for (MachineOperand *RUOp : Plan.DstUsesNeedingCopy)
+ CopyForUse[RUOp->getParent()->getParent()].insert(RUOp->getReg());
+ for (MachineInstr *RD : Plan.DstUseDefsNeedingCopy)
+ CopyForDef.insert(RD);
// Do the rewrite to allow for updated RP calculation.
const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
@@ -3069,6 +3090,14 @@ bool RewriteMFMAFormStage::rewrite(
continue;
MI->setDesc(TII->get(ReplacementOp));
+ bool Src2NeedsVGPR = Src2NeedsVGPRCache.lookup(MI);
+ // Recompute the plan instead of caching initHeuristics' result: it holds
+ // bare MI/operand pointers, so recomputing against the current MIR reflects
+ // the copies and setReg()s done for earlier candidates that may share a dst
+ // vreg (e.g. non-tied accumulation chains).
+ MFMACopyPlan Plan =
+ analyzeCopyPoints(MI, RewriteCandsSet, RewriteSrc2Regs, Src2NeedsVGPR);
+
// Case 1: insert copies for the reaching defs of the Src2Reg.
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
if (Src2->isReg()) {
@@ -3077,22 +3106,8 @@ bool RewriteMFMAFormStage::rewrite(
return false;
Register MappedReg = Src2->getReg();
- SmallVector<SlotIndex, 8> Src2ReachingDefs;
- findReachingDefs(*Src2, DAG.LIS, Src2ReachingDefs);
- SmallSetVector<MachineInstr *, 8> Src2DefsReplace;
-
- // If src2 has a use that must remain VGPR, it cannot be reclassified to
- // AGPR.
- bool Src2NeedsVGPR = Src2NeedsVGPRCache.lookup(MI);
-
- for (SlotIndex RDIndex : Src2ReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (!Src2NeedsVGPR &&
- isReachingDefAGPRForm(RD, RewriteCandsSet, RewriteSrc2Regs, *TII))
- continue;
-
- Src2DefsReplace.insert(RD);
- }
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace =
+ Plan.Src2DefsNeedingCopy;
if (!Src2DefsReplace.empty()) {
auto RI = RedefMap.find(Src2Reg);
@@ -3149,41 +3164,10 @@ bool RewriteMFMAFormStage::rewrite(
return false;
Register MappedReg = DstReg;
- SmallVector<MachineOperand *, 8> DstReachingUses;
-
- SmallVector<MachineOperand *, 8> DstReachingUseCopies;
- SmallVector<MachineInstr *, 8> DstUseDefsReplace;
-
- findReachingUses(MI, DAG.LIS, DstReachingUses);
-
- for (MachineOperand *RUOp : DstReachingUses) {
- MachineInstr *UserMI = RUOp->getParent();
- // Group members read the AGPR result directly.
- if (TII->isMAI(*UserMI) && RewriteCandsSet.contains(UserMI))
- continue;
-
- // If there is a non mai reaching use, then we need a copy.
- if (find(DstReachingUseCopies, RUOp) == DstReachingUseCopies.end())
- DstReachingUseCopies.push_back(RUOp);
-
- // Non-rewritten MAI: its defs aren't being reclassified.
- if (TII->isMAI(*UserMI))
- continue;
-
- SmallVector<SlotIndex, 8> DstUsesReachingDefs;
- findReachingDefs(*RUOp, DAG.LIS, DstUsesReachingDefs);
-
- for (SlotIndex RDIndex : DstUsesReachingDefs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(RDIndex);
- if (TII->isMAI(*RD))
- continue;
-
- // If there is a non mai reaching def of this reaching use, then we will
- // need a copy.
- if (find(DstUseDefsReplace, RD) == DstUseDefsReplace.end())
- DstUseDefsReplace.push_back(RD);
- }
- }
+ const SmallSetVector<MachineOperand *, 8> &DstReachingUseCopies =
+ Plan.DstUsesNeedingCopy;
+ const SmallSetVector<MachineInstr *, 8> &DstUseDefsReplace =
+ Plan.DstUseDefsNeedingCopy;
if (!DstUseDefsReplace.empty()) {
auto RI = RedefMap.find(DstReg);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 5909ba4d28c8c..00fb6ac007346 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -15,6 +15,8 @@
#include "GCNRegPressure.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
@@ -547,13 +549,33 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Returns true if any reaching def of src2 (\p Src2ReachingDefs) has a use
/// that requires the def to stay in VGPR form: any use that is not a COPY and
- /// not a non-excluded MFMA candidate (in \p RewriteSet but not in \p
- /// ExcludedMFMAs). Excluded MFMAs are not being rewritten, so their src2
+ /// not a rewritable MFMA candidate (in \p RewriteSet, which is already
+ /// post-exclusion). Excluded MFMAs are not being rewritten, so their src2
/// cannot be reclassified to AGPR.
bool
hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
- const SmallSetVector<MachineInstr *, 16> &RewriteSet,
- const SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet);
+
+ /// The set of copy insertion points required to rewrite a single MFMA
+ /// candidate, shared by initHeuristics (cost estimate) and rewrite (actual
+ /// insertion) so both agree on exactly which copies are needed.
+ struct MFMACopyPlan {
+ /// Non-AGPR-form reaching defs of src2 that need a bridge COPY (Case 1).
+ SmallSetVector<MachineInstr *, 8> Src2DefsNeedingCopy;
+ /// Non-group reaching uses of the dst that need a copy (Case 2).
+ SmallSetVector<MachineOperand *, 8> DstUsesNeedingCopy;
+ /// Non-MAI reaching defs of those dst uses that need a copy (Case 3).
+ SmallSetVector<MachineInstr *, 8> DstUseDefsNeedingCopy;
+ };
+
+ /// Compute the copy plan for MFMA candidate \p MI. \p GroupSet is the
+ /// post-exclusion rewrite group and \p Src2Regs the candidate src2 registers
+ /// (both used to classify AGPR-form reaching defs); \p Src2NeedsVGPR forces
+ /// every src2 reaching def to be bridged.
+ MFMACopyPlan
+ analyzeCopyPoints(MachineInstr *MI,
+ const SmallSetVector<MachineInstr *, 16> &GroupSet,
+ const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR);
public:
bool initGCNSchedStage() override;
>From 3abf235bc73333c0df9f71b1d4910d237405fc25 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 21 Jul 2026 10:42:54 +0000
Subject: [PATCH 6/6] [AMDGPU] Port MFMA-form rewrite refinements onto shared
copy-plan analysis
Extend the RewriteMFMAFormStage safe-guard and bridge-copy logic on top of
the analyzeCopyPoints copy-plan refactor:
- Safe-guard: add a post-dominator tree and isSrc2MergePoint(), and extend
hasSrc2BridgeConflict with a joint-lane coverage check and a fast path for
parallel MAI/non-MAI def pairs, excluding more unsafe src2 rewrites.
- src2 bridge: factor def classification into computeSrc2BridgePlan /
classifySrc2Bridge, which picks a merged full-width COPY vs one COPY per
def-BB from three shapes (parallel arms, sequential defs, single
non-dominating def-BB), each gated on joint-lane coverage and connectivity.
- dst bridge: add getCopyGroups() to emit the narrowest legal VGPR COPY per
subreg group.
- Occupancy: override shouldRevertScheduling to revert when VGPR/AGPR
pressure stays over the physical limit and worse than before.
Add MIR tests and regenerate sched_mfma_rewrite_copies.mir.
Co-Authored-By: Claude Opus 4.7 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 992 ++++++++++++------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 171 ++-
.../rewrite-mfma-form-pressure-refresh.mir | 101 ++
.../AMDGPU/rewrite-mfma-form-safe-guard.mir | 238 ++++-
...a-form-src2-merge-point-both-rewritten.mir | 172 +++
...fma-form-src2-mfma-plus-subreg-rewrite.mir | 134 +++
...fma-form-src2-mixed-agpr-covered-perbb.mir | 144 +++
...c2-parallel-fulldef-liveinterval-split.mir | 128 +++
...ma-form-src2-parallel-fullwidth-merged.mir | 126 +++
...fma-form-src2-seq-bypass-dominance-gap.mir | 115 ++
.../AMDGPU/sched_mfma_rewrite_copies.mir | 117 +--
11 files changed, 1937 insertions(+), 501 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-pressure-refresh.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-merge-point-both-rewritten.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mfma-plus-subreg-rewrite.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mixed-agpr-covered-perbb.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fulldef-liveinterval-split.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fullwidth-merged.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-seq-bypass-dominance-gap.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 313671f83c655..c515ad867c5ab 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1466,6 +1466,11 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
TII = ST.getInstrInfo();
SRI = ST.getRegisterInfo();
+ // rewrite() only inserts COPYs within existing blocks and never alters CFG
+ // edges, so the post-dominator tree is stable for the whole stage; compute it
+ // once here and reuse it in Check 3 and Case 1.
+ MPDT.recalculate(MF);
+
std::vector<std::pair<MachineInstr *, unsigned>> RewriteCands;
DenseMap<MachineBasicBlock *, std::set<Register>> CopyForUse;
SmallPtrSet<MachineInstr *, 8> CopyForDef;
@@ -2245,6 +2250,31 @@ bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
return Profit < ScheduleMetrics::ScaleFactor;
}
+bool RewriteMFMAFormStage::shouldRevertScheduling(unsigned WavesAfter) {
+ if (PressureAfter == PressureBefore)
+ return false;
+
+ if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
+ return true;
+
+ // This stage targets archVGPR pressure specifically, so the base
+ // mayCauseSpilling (gated on the total-RP isRegionWithExcessRP flag) may miss
+ // archVGPR excess. archVGPR and AGPR share one physical limit; revert if
+ // either dimension is still over that limit and worse than before rewrite.
+ if (RegionsWithExcessArchVGPR[RegionIdx]) {
+ unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
+ if (PressureAfter.getArchVGPRNum() > MaxArchVGPRs &&
+ PressureAfter.getArchVGPRNum() > PressureBefore.getArchVGPRNum())
+ return true;
+ if (PressureAfter.getAGPRNum() > MaxArchVGPRs &&
+ PressureAfter.getAGPRNum() > PressureBefore.getAGPRNum())
+ return true;
+ } else if (mayCauseSpilling(WavesAfter)) {
+ return true;
+ }
+ return false;
+}
+
bool ClusteredLowOccStage::shouldRevertScheduling(unsigned WavesAfter) {
if (PressureAfter == PressureBefore)
return false;
@@ -2438,6 +2468,32 @@ void RewriteMFMAFormStage::resetRewriteCandsToVGPR(
}
}
+SmallVector<RewriteMFMAFormStage::CopyGroup, 2>
+RewriteMFMAFormStage::getCopyGroups(ArrayRef<MachineOperand *> Uses,
+ Register Reg, MachineRegisterInfo &MRI,
+ const SIRegisterInfo *SRI,
+ const TargetRegisterInfo *TRI) {
+ const TargetRegisterClass *DstRC = MRI.getRegClass(Reg);
+ const TargetRegisterClass *AgprRC = SRI->getEquivalentAGPRClass(DstRC);
+ const TargetRegisterClass *FullVGPRRC = SRI->getEquivalentVGPRClass(DstRC);
+ DenseMap<unsigned, SmallVector<MachineOperand *, 4>> BySubReg;
+ for (MachineOperand *MO : Uses)
+ BySubReg[MO->getSubReg()].push_back(MO);
+ SmallVector<CopyGroup, 2> Groups;
+ for (auto &[SR, SRUses] : BySubReg) {
+ unsigned SrcSubReg = AMDGPU::NoSubRegister;
+ const TargetRegisterClass *VGPRRC = FullVGPRRC;
+ if (SR != AMDGPU::NoSubRegister)
+ if (const auto *SubRC = TRI->getSubRegisterClass(AgprRC, SR))
+ if (const auto *NarrowRC = SRI->getEquivalentVGPRClass(SubRC)) {
+ SrcSubReg = SR;
+ VGPRRC = NarrowRC;
+ }
+ Groups.push_back({SrcSubReg, VGPRRC, std::move(SRUses)});
+ }
+ return Groups;
+}
+
bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
if (!static_cast<const SIInstrInfo *>(DAG.TII)->isMAI(*MI))
return false;
@@ -2446,11 +2502,35 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
return true;
}
+bool RewriteMFMAFormStage::isSrc2MergePoint(
+ Register Src2Reg, const MachineBasicBlock *P,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &DefBlocks) const {
+ auto &MDT = DAG.LIS->getDomTree();
+
+ // (A) P post-dominates every def block -- the COPY reads a single merged
+ // %acc.
+ for (const MachineBasicBlock *DefBB : DefBlocks)
+ if (!MPDT.dominates(P, DefBB))
+ return false;
+
+ // (B) P dominates every use of Src2Reg -- %MappedReg reaches every consumer.
+ // Scanning all uses is a safe superset (non-candidate uses can only
+ // reject a rewrite, never break one).
+ for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg))
+ if (!MDT.dominates(P, UseMO.getParent()->getParent()))
+ return false;
+
+ return true;
+}
+
bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const MachineBasicBlock *CandBlock,
const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
- const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
- const {
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks,
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const DenseSet<Register> &CandSrc2Regs) const {
+ // Returns true (conflict, exclude the candidate) if any check below fires.
auto &MDT = DAG.LIS->getDomTree();
SmallVector<MachineInstr *, 8> MAIMIs;
@@ -2462,72 +2542,44 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
else
NonMAIMIs.push_back(MI);
}
-
if (NonMAIMIs.empty())
- return false; // All reaching defs are MAI; no bridge copies needed.
-
- // Check 1: MAI def dominates non-MAI def. The MFMA writes all lanes (AGPR),
- // then a non-MAI def partially overwrites some; a bridge copy there would be
- // a read-modify-write on AGPR, which the bridge mechanism cannot implement.
- bool SkipCheck2 = false;
- if (!MAIMIs.empty()) {
- for (MachineInstr *M : MAIMIs)
- for (MachineInstr *N : NonMAIMIs)
- if (MDT.dominates(M, N))
- return true;
-
- // Early-safe: if every MAI reaching def is a candidate and parallel to
- // every non-MAI def (Check 1 ruled out MAI dominating non-MAI; here the
- // reverse), then %MappedReg is defined on every path to a use -- a
- // candidate MAI gives an AGPR value directly, a non-MAI def carries a
- // bridge copy -- so Check 2 can be skipped. Check 3 is independent and
- // must still run.
- bool AllParallelAndCandidates = true;
- for (MachineInstr *M : MAIMIs) {
- if (!isRewriteCandidate(M)) {
- AllParallelAndCandidates = false;
- break;
- }
- for (MachineInstr *N : NonMAIMIs) {
- if (MDT.dominates(N, M)) {
- AllParallelAndCandidates = false;
- break;
- }
+ return false;
+
+ // Check 1: MAI def dominating a non-MAI def = AGPR RMW the bridge can't do.
+ for (MachineInstr *M : MAIMIs)
+ for (MachineInstr *N : NonMAIMIs)
+ if (MDT.dominates(M, N))
+ return true;
+
+ // SkipBridgeCoverage: skip Check 2 when every MAI def is a candidate
+ // (supplies an AGPR value) not dominated by any non-MAI def (parallel; the
+ // other direction is Check 1).
+ bool SkipBridgeCoverage =
+ !MAIMIs.empty() && all_of(MAIMIs, [&](MachineInstr *M) {
+ return isRewriteCandidate(M) &&
+ none_of(NonMAIMIs,
+ [&](MachineInstr *N) { return MDT.dominates(N, M); });
+ });
+ if (!SkipBridgeCoverage) {
+ // True if every entry->UseBlock path crosses a bridge block: walk preds,
+ // stopping at bridge blocks; reaching entry (no preds) means uncovered.
+ auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
+ if (BridgeBlocks.contains(UseBlock))
+ return true;
+ SmallPtrSet<const MachineBasicBlock *, 16> Visited;
+ SmallVector<const MachineBasicBlock *, 16> Worklist(
+ UseBlock->pred_begin(), UseBlock->pred_end());
+ while (!Worklist.empty()) {
+ const MachineBasicBlock *B = Worklist.pop_back_val();
+ if (BridgeBlocks.contains(B))
+ continue;
+ if (B->pred_empty())
+ return false;
+ if (Visited.insert(B).second)
+ Worklist.append(B->pred_begin(), B->pred_end());
}
- if (!AllParallelAndCandidates)
- break;
- }
- if (AllParallelAndCandidates)
- SkipCheck2 = true;
- }
-
- // Check 2: every entry->use path of Src2Reg must hit a bridge block (a
- // non-MAI def block where rewrite() inserts the bridge copy, or the use block
- // itself), else %MappedReg is undefined on the bypassing path. Bridge blocks
- // must *jointly* cover every path (dominance is too strict), so a backward
- // walk that reaches entry without crossing one proves an uncovered path.
- // Every use counts, incl. a plain COPY. BridgeBlocks: see
- // computeExclusionSet.
- auto CoveredByBridgeSet = [&](const MachineBasicBlock *UseBlock) {
- if (BridgeBlocks.contains(UseBlock))
return true;
- SmallPtrSet<const MachineBasicBlock *, 16> Visited;
- SmallVector<const MachineBasicBlock *, 16> Worklist(UseBlock->pred_begin(),
- UseBlock->pred_end());
- while (!Worklist.empty()) {
- const MachineBasicBlock *B = Worklist.pop_back_val();
- if (BridgeBlocks.contains(B))
- continue; // This path is covered; do not walk past the bridge block.
- if (B->pred_empty())
- return false; // Reached entry without crossing a bridge block.
- if (!Visited.insert(B).second)
- continue;
- Worklist.append(B->pred_begin(), B->pred_end());
- }
- return true;
- };
-
- if (!SkipCheck2) {
+ };
for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
const MachineBasicBlock *UseBlock = UseMO.getParent()->getParent();
if (!CoveredByBridgeSet(UseBlock))
@@ -2535,31 +2587,57 @@ bool RewriteMFMAFormStage::hasSrc2BridgeConflict(
}
}
- // Check 3: original-register connectivity. rewrite() redirects Src2Reg's
- // uses to %MappedReg, so non-AGPR-form partial subreg defs in >=2 blocks with
- // none dominating all uses split Src2Reg's live interval into disconnected
- // fragments. AGPR-form defs are absent from RedefPartialBlocks (skipped by
- // Case1), preserving the diamond joint-coverage case.
- if (RedefPartialBlocks.size() >= 2) {
- bool SingleDominator = false;
- for (const MachineBasicBlock *B : RedefPartialBlocks) {
- bool DomAllUses = true;
- for (const MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2Reg)) {
- if (!MDT.dominates(B, UseMO.getParent()->getParent())) {
- DomAllUses = false;
- break;
- }
- }
- if (DomAllUses) {
- SingleDominator = true;
- break;
- }
+ // Check 3 (fragmentation gate): a non-fragmented interval needs no merge
+ // point; a fragmented one must funnel through a single merge block.
+ if (RedefPartialBlocks.size() < 2)
+ return false;
+ auto isAcyclicSingleBlockMFMAMerge = [&]() {
+ // MAIMIs is non-empty here; only the single-non-MAI-block shape is checked.
+ SmallPtrSet<const MachineBasicBlock *, 4> NonMAIBlocks;
+ for (MachineInstr *N : NonMAIMIs)
+ NonMAIBlocks.insert(N->getParent());
+ if (NonMAIBlocks.size() != 1)
+ return false;
+ const MachineBasicBlock *NonMAIBlock = *NonMAIBlocks.begin();
+ for (MachineInstr *M : MAIMIs) {
+ const MachineBasicBlock *MBlk = M->getParent();
+ if (MBlk == CandBlock || MBlk == NonMAIBlock ||
+ !MPDT.dominates(CandBlock, MBlk))
+ return false;
}
- if (!SingleDominator)
- return true;
+ return true;
+ };
+ bool MergeOk = MAIMIs.empty()
+ ? isSrc2MergePoint(Src2Reg, CandBlock, RedefPartialBlocks)
+ : isAcyclicSingleBlockMFMAMerge();
+ if (!MergeOk)
+ return true;
+
+ // Coverage: bridge COPYs come only from non-AGPR-form reaching defs. If
+ // those don't jointly cover every live lane of Src2Reg, neither emit strategy
+ // can reconstruct the full value. No subranges = whole-reg live = no lane
+ // gap.
+ if (!DAG.LIS->hasInterval(Src2Reg))
+ return false;
+ const LiveInterval &LI = DAG.LIS->getInterval(Src2Reg);
+ if (!LI.hasSubRanges())
+ return false;
+
+ LaneBitmask LiveLanes = LaneBitmask::getNone();
+ for (const LiveInterval::SubRange &SR : LI.subranges())
+ LiveLanes |= SR.LaneMask;
+
+ LaneBitmask BridgeDefLanes = LaneBitmask::getNone();
+ for (MachineInstr *N : NonMAIMIs) {
+ if (isReachingDefAGPRForm(N, RewriteSet, CandSrc2Regs, *TII))
+ continue; // Match Src2DefsReplace's filter exactly.
+ unsigned Sub = getDefSubReg(*N, Src2Reg);
+ BridgeDefLanes |= (Sub == AMDGPU::NoSubRegister)
+ ? LaneBitmask::getAll()
+ : DAG.TRI->getSubRegIndexLaneMask(Sub);
}
- return false;
+ return (BridgeDefLanes & LiveLanes) != LiveLanes;
}
void RewriteMFMAFormStage::propagateExclusionForward(
@@ -2579,19 +2657,12 @@ void RewriteMFMAFormStage::propagateExclusionForward(
TII->getNamedOperand(*UserMI, AMDGPU::OpName::src2);
if (!UserSrc2 || !UserSrc2->isReg() || UserSrc2->getReg() != DstReg)
continue;
- if (ExcludedMFMAs.insert(UserMI).second) {
- LLVM_DEBUG(dbgs() << "[initHeuristics] exclude downstream MFMA "
- "(src2 = excluded MFMA dst): "
- << *UserMI);
+ if (ExcludedMFMAs.insert(UserMI).second)
Worklist.push_back(UserMI);
- }
}
}
}
-// rewrite() reclassifies DstReg to AGPR so the MFMA writes its result directly
-// into AGPR (no post-MFMA copy). That is legal only if every def and use of
-// DstReg across its live range supports AGPR-class operands; check that here.
bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
MachineInstr *MFMA) {
SmallVector<MachineOperand *, 8> DstReachingUses;
@@ -2599,9 +2670,7 @@ bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
SmallPtrSet<MachineInstr *, 8> CheckedDefs;
SmallVector<MachineInstr *, 4> SafeSubregDefs;
- // Phase 1: classify subreg reaching defs.
- // Non-agnostic subreg def → immediate conflict.
- // Agnostic (COPY/AV_MOV) subreg def → defer to orphan-use check.
+ // Phase 1: non-agnostic subreg def → conflict; agnostic → defer to Phase 2.
for (MachineOperand *RUOp : DstReachingUses) {
SmallVector<SlotIndex, 8> ReachingDefs;
findReachingDefs(*RUOp, DAG.LIS, ReachingDefs);
@@ -2617,9 +2686,8 @@ bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
}
}
- // Phase 2: agnostic subreg defs lower to v_accvgpr_write after reclassify,
- // so their orphan uses read an AGPR sub-register. Only COPY and AV_MOV can
- // legally source an AGPR sub-register; anything else is a conflict.
+ // Phase 2: an agnostic subreg def's lane becomes AGPR after reclassify, so a
+ // non-agnostic use of it can't legally source that AGPR sub-register lane.
for (MachineInstr *RD : SafeSubregDefs) {
SlotIndex RDIdx = DAG.LIS->getInstructionIndex(*RD);
for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(DstReg)) {
@@ -2640,120 +2708,95 @@ bool RewriteMFMAFormStage::hasDstSubregConflict(Register DstReg,
SmallPtrSet<MachineInstr *, 16> RewriteMFMAFormStage::computeExclusionSet(
const SmallSetVector<MachineInstr *, 16> &RewriteSet) {
- // Per-MI checks, cheapest-first: (1) hasSrc2BridgeConflict, then
- // (2) hasDstSubregConflict (skipped when 1 already excludes). Exclusion
- // propagates forward (dst->src2 via propagateExclusionForward) and backward
- // (MAI reaching-defs of a conflicted src2).
+ // Per-MI checks, cheapest-first: hasSrc2BridgeConflict before
+ // hasDstSubregConflict.
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs;
- // Per src2 register, union reaching-def blocks across all candidates (one
- // candidate sees only one path): non-MFMA -> Src2BridgeBlocks (Check 2), plus
- // non-AGPR-form partial-subreg -> Src2RedefPartialBlocks (Check 3). Reaching-
- // def (not all-def) keeps bypass-killed and entry IMPLICIT_DEFs out.
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2BridgeBlocks;
DenseMap<Register, SmallPtrSet<const MachineBasicBlock *, 8>>
Src2RedefPartialBlocks;
- // Rebuild the block maps from the still-live candidates \p Active: excluding
- // a candidate makes its src2/MAI def non-AGPR-form (isReachingDefAGPRForm
- // reads Active), which can add new Check 3 redef sources -- so the full
- // pre-exclusion set would miss them.
- auto recomputeBlocks = [&](const SmallSetVector<MachineInstr *, 16> &Active) {
- Src2BridgeBlocks.clear();
- Src2RedefPartialBlocks.clear();
- DenseSet<Register> CandSrc2Regs;
- for (MachineInstr *MI : Active) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
- for (MachineInstr *MI : Active) {
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- if (!Src2 || !Src2->isReg())
+ // src2 reaching defs per candidate, computed in Phase 2 and reused in
+ // Phase 3.
+ DenseMap<MachineInstr *, SmallVector<SlotIndex, 8>> Src2DefsCache;
+
+ // === Phase 1: gather candidate src2 regs ===
+ // Consumed by isReachingDefAGPRForm; must be complete before Phase 2 uses it,
+ // hence its own pass.
+ DenseSet<Register> CandSrc2Regs;
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
+ }
+
+ // === Phase 2: build per-src2 block maps ===
+ // Union across candidates: one candidate sees only one reaching path.
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2 || !Src2->isReg())
+ continue;
+ Register Src2Reg = Src2->getReg();
+ auto &Blocks = Src2BridgeBlocks[Src2Reg];
+ auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
+ SmallVector<SlotIndex, 8> &Src2Defs = Src2DefsCache[MI];
+ findReachingDefs(*Src2, DAG.LIS, Src2Defs);
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
+ if (!RD)
continue;
- Register Src2Reg = Src2->getReg();
- auto &Blocks = Src2BridgeBlocks[Src2Reg];
- auto &RedefBlocks = Src2RedefPartialBlocks[Src2Reg];
- SmallVector<SlotIndex, 8> Src2Defs;
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI);
- if (!RD || TII->isMFMA(*RD))
- continue;
- Blocks.insert(RD->getParent());
- // Check 3 fragmentation source: non-AGPR-form partial-subreg defs only
- // (AGPR-form skipped by Case1; full-width defs stay connected).
- if (!isReachingDefAGPRForm(RD, Active, CandSrc2Regs, *TII) &&
+ if (TII->isMFMA(*RD)) {
+ // Fragmentation source: an MAI def is an independent value source into
+ // Src2Reg, so it counts toward Check 3's interval-fragmentation test.
+ RedefBlocks.insert(RD->getParent());
+ } else {
+ Blocks.insert(RD->getParent()); // Bridge-coverage source.
+ if (!isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, *TII) &&
getDefSubReg(*RD, Src2Reg) != AMDGPU::NoSubRegister)
- RedefBlocks.insert(RD->getParent());
+ RedefBlocks.insert(RD->getParent()); // Fragmentation source.
}
}
- };
-
- // Fixpoint: each exclusion can expose new conflicts in others (see
- // recomputeBlocks). Iterate to convergence; exclusions only grow over a
- // finite candidate set, so this terminates.
- bool Changed;
- do {
- Changed = false;
-
- SmallSetVector<MachineInstr *, 16> Active;
- for (MachineInstr *MI : RewriteSet)
- if (!ExcludedMFMAs.count(MI))
- Active.insert(MI);
-
- recomputeBlocks(Active);
+ }
- for (MachineInstr *MI : Active) {
- // May have been excluded by forward/backward propagation earlier this
- // round.
- if (ExcludedMFMAs.count(MI))
- continue;
+ // === Phase 3: per-MI conflict (from the block maps above) + propagation ===
+ for (MachineInstr *MI : RewriteSet) {
+ // May have been excluded by forward/backward propagation earlier in this
+ // pass.
+ if (ExcludedMFMAs.count(MI))
+ continue;
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- Register DstReg = MI->getOperand(0).getReg();
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ Register DstReg = MI->getOperand(0).getReg();
- bool HasConflict = false;
- SmallVector<SlotIndex, 8> Src2Defs;
- if (Src2->isReg()) {
- findReachingDefs(*Src2, DAG.LIS, Src2Defs);
- HasConflict = hasSrc2BridgeConflict(
- Src2Defs, Src2->getReg(), Src2BridgeBlocks[Src2->getReg()],
- Src2RedefPartialBlocks[Src2->getReg()]);
- }
- bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
+ bool HasConflict = false;
+ ArrayRef<SlotIndex> Src2Defs;
+ if (Src2->isReg()) {
+ Src2Defs = Src2DefsCache[MI]; // Reuse the Phase 2 computation.
+ HasConflict = hasSrc2BridgeConflict(
+ Src2Defs, Src2->getReg(), MI->getParent(),
+ Src2BridgeBlocks[Src2->getReg()],
+ Src2RedefPartialBlocks[Src2->getReg()], RewriteSet, CandSrc2Regs);
+ }
+ bool HasDstSubregDef = !HasConflict && hasDstSubregConflict(DstReg, MI);
- if (!HasConflict && !HasDstSubregDef)
- continue;
+ if (!HasConflict && !HasDstSubregDef)
+ continue;
- if (ExcludedMFMAs.insert(MI).second) {
- LLVM_DEBUG(
- dbgs() << "[computeExclusionSet] exclude MFMA ("
- << (HasConflict ? "src2 dominance conflict" : "")
- << (HasConflict && HasDstSubregDef ? " + " : "")
- << (HasDstSubregDef ? "dst non-MAI subreg overwrite" : "")
- << "): " << *MI);
- propagateExclusionForward(MI, ExcludedMFMAs);
- Changed = true;
- }
+ if (ExcludedMFMAs.insert(MI).second)
+ propagateExclusionForward(MI, ExcludedMFMAs);
- // Backward: exclude MAI reaching-defs that are themselves candidates.
- if (!HasConflict)
+ // Backward: exclude MAI reaching-defs that are themselves candidates.
+ if (!HasConflict)
+ continue;
+ for (SlotIndex SI : Src2Defs) {
+ MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
+ if (!TII->isMFMA(*DefMI) || !isRewriteCandidate(DefMI) ||
+ !ExcludedMFMAs.insert(DefMI).second)
continue;
- for (SlotIndex SI : Src2Defs) {
- MachineInstr *DefMI = DAG.LIS->getInstructionFromIndex(SI);
- if (!TII->isMFMA(*DefMI) || !isRewriteCandidate(DefMI) ||
- !ExcludedMFMAs.insert(DefMI).second)
- continue;
- LLVM_DEBUG(dbgs() << "[computeExclusionSet] exclude MAI def "
- "(backward from src2 dominance conflict): "
- << *DefMI);
- propagateExclusionForward(DefMI, ExcludedMFMAs);
- Changed = true;
- }
+ propagateExclusionForward(DefMI, ExcludedMFMAs);
}
- } while (Changed);
+ }
return ExcludedMFMAs;
}
@@ -2763,9 +2806,9 @@ RewriteMFMAFormStage::MFMACopyPlan RewriteMFMAFormStage::analyzeCopyPoints(
const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR) {
MFMACopyPlan Plan;
- // Case 1: the reaching defs of the src2 operand that need a bridge copy.
- // If src2 stays VGPR, every reaching def needs a copy; otherwise reaching
- // defs already in AGPR form (group members, av-movs, group copies) don't.
+ // Case 1: src2 reaching defs needing a bridge copy. If Src2NeedsVGPR, every
+ // def needs one; else AGPR-form defs (group members, av-movs, group copies)
+ // don't.
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
if (Src2->isReg()) {
SmallVector<SlotIndex, 8> Src2ReachingDefs;
@@ -2821,15 +2864,12 @@ bool RewriteMFMAFormStage::initHeuristics(
if (isRewriteCandidate(&MI))
RewriteSet.insert(&MI);
- // Remove unsafe candidates so RewriteSet becomes the post-exclusion group --
- // both the Pass 2 worklist and the "will be AGPR-form" set queried
- // downstream.
+ // Remove unsafe candidates so RewriteSet is the post-exclusion group -- both
+ // the Pass 2 worklist and the "will be AGPR-form" set queried downstream.
SmallPtrSet<MachineInstr *, 16> ExcludedMFMAs =
computeExclusionSet(RewriteSet);
- for (MachineInstr *Excluded : ExcludedMFMAs) {
- LLVM_DEBUG(dbgs() << "[initHeuristics] skip excluded MFMA: " << *Excluded);
+ for (MachineInstr *Excluded : ExcludedMFMAs)
RewriteSet.remove(Excluded);
- }
// src2 regs of the post-exclusion group only: an excluded candidate's src2
// stays VGPR, so isReachingDefAGPRForm must not treat a COPY from it as
@@ -2841,6 +2881,10 @@ bool RewriteMFMAFormStage::initHeuristics(
CandSrc2Regs.insert(Src2->getReg());
}
+ // rewrite() emits src2 bridge COPYs once per Src2Reg (its first candidate);
+ // charge only that first candidate too, matching what rewrite() emits.
+ DenseSet<Register> ChargedSrc2Regs;
+
// Pass 2: compute heuristics for the remaining (rewritable) candidates.
for (MachineInstr *MI : RewriteSet) {
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
@@ -2864,17 +2908,21 @@ bool RewriteMFMAFormStage::initHeuristics(
MFMACopyPlan Plan =
analyzeCopyPoints(MI, RewriteSet, CandSrc2Regs, Src2NeedsVGPR);
- if (Src2->isReg()) {
- for (MachineInstr *RD : Plan.Src2DefsNeedingCopy) {
- // If this reaching def writes into the src2 register itself (possibly
- // a partial subreg write), the def will be reclassified together with
- // src2 and does not need a bridge copy.
- if (any_of(RD->defs(), [&](const MachineOperand &DefOp) {
- return DefOp.isReg() && DefOp.getReg() == Src2->getReg();
- }))
- continue;
- CopyForDef.insert(RD);
- }
+ if (Src2->isReg() && !Plan.Src2DefsNeedingCopy.empty() &&
+ ChargedSrc2Regs.insert(Src2->getReg()).second) {
+ // Charge exactly the COPYs rewrite()'s Case 1 will emit for this
+ // Src2Reg's first bridging candidate: the merged path is one COPY before
+ // the MFMA (charge MI -- its block matches, and the dst class
+ // approximates the src2-width COPY closely enough for this
+ // spill-estimate); the per-BB path is one COPY per def-BB after its last
+ // def (charge each BB's LastRD).
+ Src2BridgeDecision D =
+ classifySrc2Bridge(MI, Src2, Plan.Src2DefsNeedingCopy);
+ if (D.UseMerged)
+ CopyForDef.insert(MI);
+ else
+ for (MachineBasicBlock *BB : D.Plan.SortedBBs)
+ CopyForDef.insert(D.Plan.PerBBMap.find(BB)->second.LastRD);
}
MachineOperand &Dst = MI->getOperand(0);
@@ -3006,6 +3054,225 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
return Cost + CopyCost;
}
+RewriteMFMAFormStage::Src2BridgePlan
+RewriteMFMAFormStage::computeSrc2BridgePlan(
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace, Register Src2Reg,
+ const MachineBasicBlock *UseMBB) {
+ Src2BridgePlan Plan;
+
+ // Phase 1: live lanes of src2 -- union of sub-ranges, or all lanes if the reg
+ // has none (else AllLanesCovered reads as spuriously uncovered for full-width
+ // src2).
+ if (DAG.LIS->hasInterval(Src2Reg)) {
+ const LiveInterval &Src2LI = DAG.LIS->getInterval(Src2Reg);
+ if (Src2LI.hasSubRanges())
+ for (const LiveInterval::SubRange &SR : Src2LI.subranges())
+ Plan.Src2LiveLanes |= SR.LaneMask;
+ else
+ Plan.Src2LiveLanes = LaneBitmask::getAll();
+ }
+
+ // Phase 2: fold each reaching def into per-BB state (lanes written, last def)
+ // and accumulate the total lanes defined across all BBs.
+ LaneBitmask TotalDefLanes = LaneBitmask::getNone();
+ for (MachineInstr *RD : Src2DefsReplace) {
+ unsigned SubReg = getDefSubReg(*RD, Src2Reg);
+ if (SubReg != AMDGPU::NoSubRegister)
+ Plan.HasSubregDef = true;
+ LaneBitmask Lanes = (SubReg == AMDGPU::NoSubRegister)
+ ? LaneBitmask::getAll()
+ : DAG.TRI->getSubRegIndexLaneMask(SubReg);
+ TotalDefLanes |= Lanes;
+ Src2PerBBEntry &Entry = Plan.PerBBMap[RD->getParent()];
+ Entry.UnionLanes |= Lanes;
+ if (!Entry.LastRD || DAG.LIS->getInstructionIndex(*Entry.LastRD) <
+ DAG.LIS->getInstructionIndex(*RD))
+ Entry.LastRD = RD;
+ }
+
+ // Phase 3: do the defs jointly cover every live lane of src2?
+ Plan.AllLanesCovered =
+ Plan.Src2LiveLanes.any() &&
+ (TotalDefLanes & Plan.Src2LiveLanes) == Plan.Src2LiveLanes;
+
+ // Phase 4: order def-BBs by their last-def slot, then test whether they form
+ // a dominator chain (each dominates the next) -- the sequential-bridge path.
+ for (auto &[BB, Entry] : Plan.PerBBMap)
+ Plan.SortedBBs.push_back(BB);
+ llvm::sort(Plan.SortedBBs, [&](MachineBasicBlock *A, MachineBasicBlock *B) {
+ return DAG.LIS->getInstructionIndex(*Plan.PerBBMap[A].LastRD) <
+ DAG.LIS->getInstructionIndex(*Plan.PerBBMap[B].LastRD);
+ });
+ auto &MDT = DAG.LIS->getDomTree();
+ Plan.IsSequential = true;
+ for (unsigned I = 0; I + 1 < Plan.SortedBBs.size(); ++I)
+ if (!MDT.dominates(Plan.SortedBBs[I], Plan.SortedBBs[I + 1])) {
+ Plan.IsSequential = false;
+ break;
+ }
+
+ // The per-BB (sequential) bridge chain only works if the deepest def-BB
+ // dominates the use: each per-BB COPY consumes Src2Reg inside its own def-BB,
+ // so if some path reaches the use without passing through the deepest def-BB
+ // (e.g. a bypass edge, or parallel def-BBs that each fully define Src2Reg),
+ // Src2Reg's sub-defs no longer share the merge-point use that kept its live
+ // interval connected -> "Multiple connected components". Fall back to the
+ // merged (single full-width COPY before the MFMA) path in that case.
+ if (Plan.IsSequential && !Plan.SortedBBs.empty() && UseMBB &&
+ !MDT.dominates(Plan.SortedBBs.back(), UseMBB))
+ Plan.IsSequential = false;
+
+ return Plan;
+}
+
+RewriteMFMAFormStage::Src2BridgeDecision
+RewriteMFMAFormStage::classifySrc2Bridge(
+ MachineInstr *MI, MachineOperand *Src2,
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace) {
+ Src2BridgeDecision D;
+ D.Plan =
+ computeSrc2BridgePlan(Src2DefsReplace, Src2->getReg(), MI->getParent());
+
+ SmallVector<SlotIndex, 8> Src2AllDefs;
+ findReachingDefs(*Src2, DAG.LIS, Src2AllDefs);
+ bool HasMAIReachingDef = false;
+ for (SlotIndex SI : Src2AllDefs)
+ if (MachineInstr *RD = DAG.LIS->getInstructionFromIndex(SI))
+ if (TII->isMFMA(*RD))
+ HasMAIReachingDef = true;
+
+ // per-BB bridging (one COPY per def-BB) is the default; three shapes below
+ // need a merged full-width COPY instead. Otherwise (a single dominating
+ // def-BB, or none) per-BB is always safe.
+ bool ParallelMultiArm = D.Plan.SortedBBs.size() >= 2 && !D.Plan.IsSequential;
+
+ const MachineBasicBlock *CandBlock = MI->getParent();
+ bool PerBBSafe = true;
+ if (ParallelMultiArm) {
+ // Shape 1 -- parallel arms (>=2 mutually non-dominating def-BBs). Safe
+ // only if
+ // 1. Lanes: the arms jointly cover every live lane (AllLanesCovered),
+ // else
+ // MappedReg is left partly undefined; and
+ // 2. Connectivity: some non-implicit use dominated by CandBlock (past the
+ // merge point, so every arm reaches it) survives to keep Src2Reg
+ // connected -- otherwise redirecting the candidate uses to MappedReg
+ // splits its interval per arm ("Multiple connected components").
+ bool RemainingUseConnects = false;
+ if (D.Plan.AllLanesCovered) {
+ auto &MDT = DAG.LIS->getDomTree();
+ for (MachineOperand &UseMO : DAG.MRI.use_nodbg_operands(Src2->getReg())) {
+ if (UseMO.isImplicit())
+ continue;
+ MachineInstr *UseMI = UseMO.getParent();
+ // A src2 use of any MFMA on this reg is redirected to MappedReg
+ // (isRewriteCandidate is state-dependent -- already-rewritten MFMAs
+ // report false -- so match on the src2 operand, aligned with
+ // ReplaceMap).
+ if (TII->isMFMA(*UseMI) &&
+ TII->getNamedOperand(*UseMI, AMDGPU::OpName::src2) == &UseMO)
+ continue;
+ if (!MDT.dominates(CandBlock, UseMI->getParent()))
+ continue;
+ RemainingUseConnects = true;
+ break;
+ }
+ }
+ PerBBSafe = D.Plan.AllLanesCovered && RemainingUseConnects;
+ } else if (D.Plan.SortedBBs.size() >= 2) {
+ // Shape 2 -- sequential multi-BB defs (dominator chain): the per-def-BB
+ // COPYs must jointly cover every live lane.
+ PerBBSafe = D.Plan.AllLanesCovered;
+ } else if (D.Plan.SortedBBs.size() == 1 && !D.Plan.IsSequential) {
+ // Shape 3 -- single def-BB not dominating the use: a per-BB COPY leaves
+ // MappedReg undefined on the bypass path, so fall back to merged -- unless
+ // a MAI reaching def forces per-BB (merged would read Src2Reg after its
+ // MFMA def is renamed to AGPR form; the upstream coverage gate keeps per-BB
+ // safe here).
+ PerBBSafe = HasMAIReachingDef;
+ }
+
+ D.UseMerged = !PerBBSafe;
+ return D;
+}
+
+void RewriteMFMAFormStage::emitMergedSrc2Bridge(
+ MachineInstr *MI, Register Src2Reg, Register MappedReg,
+ function_ref<void(MachineInstr *, MachineInstrBuilder &)>
+ UpdateRegionBefore) {
+ // At the MFMA the LIS PHI has merged all paths, so every sub-range is live
+ // and the full-width COPY reads a fully-defined Src2Reg.
+ MachineInstrBuilder VGPRCopy =
+ BuildMI(*MI->getParent(), MI->getIterator(), MI->getDebugLoc(),
+ TII->get(TargetOpcode::COPY))
+ .addDef(MappedReg, {}, 0)
+ .addUse(Src2Reg, {}, 0);
+ DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
+ UpdateRegionBefore(MI, VGPRCopy);
+}
+
+void RewriteMFMAFormStage::emitPerDefBBSrc2Bridges(
+ Register Src2Reg, Register MappedReg, const Src2BridgePlan &Plan,
+ SmallPtrSetImpl<MachineInstr *> &ReachingDefCopies,
+ function_ref<void(MachineInstr *, MachineInstrBuilder &)>
+ UpdateRegionAfter) {
+ // One bridge COPY per def-BB, after its last reaching def. Def-BBs form a
+ // dominator chain (Plan.IsSequential), so each COPY dominates the MFMA.
+ //
+ // A full-width def overwrites upstream same-reg defs, so multiple def-BBs
+ // must be complementary sub-reg writes — enforced by the assert below.
+ assert((Plan.SortedBBs.size() <= 1 || Plan.AllLanesCovered) &&
+ "sequential multi-BB src2 bridges must jointly cover the live range");
+
+ // Lane mask -> the sub-reg index exactly matching it and valid in the AGPR
+ // class; NoSubRegister if none.
+ const TargetRegisterClass *AgprRC =
+ SRI->getEquivalentAGPRClass(DAG.MRI.getRegClass(Src2Reg));
+ auto findSubRegForLanes = [&](LaneBitmask Lanes) -> unsigned {
+ for (unsigned SubIdx = 1; SubIdx < DAG.TRI->getNumSubRegIndices();
+ ++SubIdx) {
+ if (DAG.TRI->getSubRegIndexLaneMask(SubIdx) != Lanes)
+ continue;
+ if (AgprRC && DAG.TRI->getSubClassWithSubReg(AgprRC, SubIdx) == nullptr)
+ continue;
+ return SubIdx;
+ }
+ return AMDGPU::NoSubRegister;
+ };
+
+ bool IsFirstSubRegCopy = true;
+ for (MachineBasicBlock *BB : Plan.SortedBBs) {
+ const Src2PerBBEntry &Entry = Plan.PerBBMap.find(BB)->second;
+ MachineInstr *RD = Entry.LastRD;
+ if (!ReachingDefCopies.insert(RD).second)
+ continue;
+
+ // Narrow to a sub-reg COPY only when the defs jointly cover the range but
+ // this BB writes just part of it; otherwise (incl. AllLanesCovered==0) stay
+ // full-width, so no MappedReg lane is left undefined.
+ unsigned CopySrcSubReg = AMDGPU::NoSubRegister;
+ if (Plan.AllLanesCovered &&
+ (Entry.UnionLanes & Plan.Src2LiveLanes) != Plan.Src2LiveLanes)
+ CopySrcSubReg = findSubRegForLanes(Entry.UnionLanes);
+
+ // The first sub-reg write into the fresh MappedReg is flagged undef to
+ // establish its full live range; later sub-reg writes fill in the rest.
+ RegState DefFlags = RegState(0);
+ if (CopySrcSubReg != AMDGPU::NoSubRegister && IsFirstSubRegCopy) {
+ DefFlags = RegState::Undef;
+ IsFirstSubRegCopy = false;
+ }
+
+ MachineInstrBuilder VGPRCopy =
+ BuildMI(*RD->getParent(), std::next(RD->getIterator()),
+ RD->getDebugLoc(), TII->get(TargetOpcode::COPY))
+ .addDef(MappedReg, DefFlags, CopySrcSubReg)
+ .addUse(Src2Reg, {}, CopySrcSubReg);
+ DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
+ UpdateRegionAfter(RD, VGPRCopy);
+ }
+}
+
bool RewriteMFMAFormStage::rewrite(
ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands) {
DenseMap<MachineInstr *, unsigned> FirstMIToRegion;
@@ -3034,14 +3301,15 @@ bool RewriteMFMAFormStage::rewrite(
// operands, 2. the reaching uses of the dst operands, and 3. the reaching
// defs of the reaching uses of the dst operand.
//
- // In the first case, we simply insert copies after each of the reaching
- // definitions. In the second case, we collect all the uses of a given dest
- // and organize them by MBB. Then, we insert 1 copy for each MBB before the
+ // In the first case, we insert one copy per def-BB after that BB's last
+ // reaching def (or, for parallel BBs with subreg defs, a single copy before
+ // the MFMA). In the second case, we collect all the uses of a given dest and
+ // organize them by MBB. Then, we insert 1 copy for each MBB before the
// earliest use. Since the use may have multiple reaching defs, and since we
// want to replace the register it is using with the result of the copy, we
- // must handle case 3. In the third case, we simply insert a copy after each
- // of the reaching defs to connect to the copy of the reaching uses of the dst
- // reg. This allows us to avoid inserting copies next to the MFMAs.
+ // must handle case 3. In the third case, we insert a copy after the reaching
+ // defs to connect to the copy of the reaching uses of the dst reg, grouped by
+ // sub-reg. This allows us to avoid inserting copies next to the MFMAs.
//
// While inserting the copies, we maintain a map of operands which will use
// different regs (i.e. the result of the copies). For example, a case 1 src2
@@ -3073,8 +3341,36 @@ bool RewriteMFMAFormStage::rewrite(
DenseMap<unsigned, DenseMap<Register, SmallPtrSet<MachineOperand *, 8>>>
ReachingUseTracker;
- // Collect the candidate group; its members share AGPR-form operands
- // post-rewrite, so reaching defs feeding any member need no bridge copy.
+ // Bilateral region-boundary helpers (used by Case 1, 2, and 3).
+ auto UpdateRegionBefore = [&](MachineInstr *UseInst,
+ MachineInstrBuilder &Copy) {
+ auto FI = FirstMIToRegion.find(UseInst);
+ if (FI != FirstMIToRegion.end()) {
+ DAG.Regions[FI->second].first = Copy;
+ FirstMIToRegion.erase(UseInst);
+ auto LMI = LastMIToRegion.find(UseInst);
+ if (LMI != LastMIToRegion.end()) {
+ DAG.Regions[LMI->second].second = Copy;
+ LastMIToRegion.erase(UseInst);
+ }
+ }
+ };
+ auto UpdateRegionAfter = [&](MachineInstr *DefInst,
+ MachineInstrBuilder &Copy) {
+ if (LastMIToRegion.contains(DefInst)) {
+ DAG.Regions[LastMIToRegion[DefInst]].second = Copy;
+ LastMIToRegion.erase(DefInst);
+ auto FI2 = FirstMIToRegion.find(DefInst);
+ if (FI2 != FirstMIToRegion.end()) {
+ DAG.Regions[FI2->second].first = Copy;
+ FirstMIToRegion.erase(DefInst);
+ }
+ }
+ };
+
+ // Collect the candidate group (post-exclusion); its members share AGPR-form
+ // operands post-rewrite, so reaching defs feeding any member need no bridge
+ // copy.
SmallSetVector<MachineInstr *, 16> RewriteCandsSet;
DenseSet<Register> RewriteSrc2Regs;
for (auto &[MI, OriginalOpcode] : RewriteCands) {
@@ -3111,7 +3407,8 @@ bool RewriteMFMAFormStage::rewrite(
if (!Src2DefsReplace.empty()) {
auto RI = RedefMap.find(Src2Reg);
- if (RI != RedefMap.end()) {
+ bool NewMapping = RI == RedefMap.end();
+ if (!NewMapping) {
MappedReg = RI->second;
} else {
assert(!ReachingDefCopyMap.contains(Src2Reg));
@@ -3124,26 +3421,18 @@ bool RewriteMFMAFormStage::rewrite(
RedefMap[Src2Reg] = MappedReg;
}
- // If none exists, create a copy from this reaching def.
- // We may have inserted a copy already in an earlier iteration.
- for (MachineInstr *RD : Src2DefsReplace) {
- // Do not create redundant copies.
- if (ReachingDefCopyMap[Src2Reg].insert(RD).second) {
- MachineInstrBuilder VGPRCopy =
- BuildMI(*RD->getParent(), std::next(RD->getIterator()),
- RD->getDebugLoc(), TII->get(TargetOpcode::COPY))
- .addDef(MappedReg, {}, 0)
- .addUse(Src2Reg, {}, 0);
- DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
-
- // If this reaching def was the last MI in the region, update the
- // region boundaries.
- if (LastMIToRegion.contains(RD)) {
- unsigned UpdateRegion = LastMIToRegion[RD];
- DAG.Regions[UpdateRegion].second = VGPRCopy;
- LastMIToRegion.erase(RD);
- }
- }
+ // Emit the bridge COPYs once per Src2Reg, when its mapping is first
+ // created (strategy chosen below); later sharers reuse the same
+ // MappedReg. Re-emitting would define MappedReg twice, splitting its
+ // live interval into multiple connected components.
+ if (NewMapping) {
+ Src2BridgeDecision D = classifySrc2Bridge(MI, Src2, Src2DefsReplace);
+ if (D.UseMerged)
+ emitMergedSrc2Bridge(MI, Src2Reg, MappedReg, UpdateRegionBefore);
+ else
+ emitPerDefBBSrc2Bridges(Src2Reg, MappedReg, D.Plan,
+ ReachingDefCopyMap[Src2Reg],
+ UpdateRegionAfter);
}
}
@@ -3164,76 +3453,108 @@ bool RewriteMFMAFormStage::rewrite(
return false;
Register MappedReg = DstReg;
+ SmallVector<MachineOperand *, 4> SameBlockUses; // same-BB Case2 uses
+
+ // Case 2 uses and Case 3 defs come from the shared coarse plan.
const SmallSetVector<MachineOperand *, 8> &DstReachingUseCopies =
Plan.DstUsesNeedingCopy;
const SmallSetVector<MachineInstr *, 8> &DstUseDefsReplace =
Plan.DstUseDefsNeedingCopy;
if (!DstUseDefsReplace.empty()) {
- auto RI = RedefMap.find(DstReg);
- if (RI != RedefMap.end()) {
- MappedReg = RI->second;
- } else {
- assert(!ReachingDefCopyMap.contains(DstReg));
- const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(DstReg);
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(DstRC);
+ // Reclassify DstReg to AGPR directly when only subreg COPY/AV_MOV writes
+ // reach here (Strategy A); a full-width non-MAI def instead needs a VGPR
+ // snapshot (MappedReg) via bridge COPYs (Strategy B). Non-agnostic subreg
+ // writers are gated out upstream by HasDstSubregDef in initHeuristics.
+ bool HasFullWidthDef = any_of(DstUseDefsReplace, [&](MachineInstr *RD) {
+ return getDefSubReg(*RD, DstReg) == AMDGPU::NoSubRegister;
+ });
+ if (HasFullWidthDef) {
+ auto RI = RedefMap.find(DstReg);
+ if (RI != RedefMap.end()) {
+ MappedReg = RI->second;
+ } else {
+ assert(!ReachingDefCopyMap.contains(DstReg));
+ const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(DstReg);
+ const TargetRegisterClass *VGPRRC =
+ SRI->getEquivalentVGPRClass(DstRC);
- // Track the mapping of the original register to the new register.
- MappedReg = DAG.MRI.createVirtualRegister(VGPRRC);
- RedefMap[DstReg] = MappedReg;
- }
+ // Track the mapping of the original register to the new register.
+ MappedReg = DAG.MRI.createVirtualRegister(VGPRRC);
+ RedefMap[DstReg] = MappedReg;
+ }
- // If none exists, create a copy from this reaching def.
- // We may have inserted a copy already in an earlier iteration.
- for (MachineInstr *RD : DstUseDefsReplace) {
- // Do not create reundant copies.
- if (ReachingDefCopyMap[DstReg].insert(RD).second) {
+ // If none exists, create a copy from this reaching def.
+ // We may have inserted a copy already in an earlier iteration.
+ for (MachineInstr *RD : DstUseDefsReplace) {
+ // Do not create redundant copies.
+ if (!ReachingDefCopyMap[DstReg].insert(RD).second)
+ continue;
+ unsigned SubReg = getDefSubReg(*RD, DstReg);
MachineInstrBuilder VGPRCopy =
BuildMI(*RD->getParent(), std::next(RD->getIterator()),
RD->getDebugLoc(), TII->get(TargetOpcode::COPY))
- .addDef(MappedReg, {}, 0)
- .addUse(DstReg, {}, 0);
+ .addDef(MappedReg,
+ SubReg != AMDGPU::NoSubRegister ? RegState::Undef
+ : RegState(0),
+ SubReg)
+ .addUse(DstReg, {}, SubReg);
DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
-
- // If this reaching def was the last MI in the region, update the
- // region boundaries.
- auto LMI = LastMIToRegion.find(RD);
- if (LMI != LastMIToRegion.end()) {
- unsigned UpdateRegion = LMI->second;
- DAG.Regions[UpdateRegion].second = VGPRCopy;
- LastMIToRegion.erase(RD);
- }
+ UpdateRegionAfter(RD, VGPRCopy);
}
}
}
DenseSet<MachineOperand *> &DstRegSet = ReplaceMap[DstReg];
+ // Same-BB uses deferred to getCopyGroups batch (dedup + narrow).
+ // Cross-BB uses deferred to ReachingUseTracker batch.
for (MachineOperand *RU : DstReachingUseCopies) {
MachineBasicBlock *RUBlock = RU->getParent()->getParent();
- // Just keep track of the reaching use of this register by block. After we
- // have scanned all the MFMAs we can find optimal insert pts.
+ unsigned BBNum = RUBlock->getNumber();
if (RUBlock != MI->getParent()) {
- ReachingUseTracker[RUBlock->getNumber()][DstReg].insert(RU);
+ LLVM_DEBUG(dbgs() << "[RewriteMFMAForm] Case2 enqueue (cross-BB): Dst="
+ << printReg(DstReg, SRI) << " use in BB#" << BBNum
+ << ": " << *RU->getParent());
+ ReachingUseTracker[BBNum][DstReg].insert(RU);
continue;
}
+ SameBlockUses.push_back(RU);
+ }
- // Special case, the use is in the same block as the MFMA. Insert the copy
- // just before the use.
- const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(DstReg);
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(DstRC);
- Register NewUseReg = DAG.MRI.createVirtualRegister(VGPRRC);
- MachineInstr *UseInst = RU->getParent();
- MachineInstrBuilder VGPRCopy =
- BuildMI(*UseInst->getParent(), UseInst->getIterator(),
- UseInst->getDebugLoc(), TII->get(TargetOpcode::COPY))
- .addDef(NewUseReg, {}, 0)
- .addUse(DstReg, {}, 0);
- DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
- // Since we know this use has only one reaching def, we can replace the
- // use reg.
- RU->setReg(NewUseReg);
- // Track the copy source operand for r eplacement.
- DstRegSet.insert(&VGPRCopy->getOperand(1));
+ if (!SameBlockUses.empty()) {
+ MachineInstr *UseInst =
+ (*llvm::min_element(SameBlockUses, [&](MachineOperand *A,
+ MachineOperand *B) {
+ return SlotIndex::isEarlierInstr(
+ DAG.LIS->getInstructionIndex(*A->getParent()),
+ DAG.LIS->getInstructionIndex(*B->getParent()));
+ }))->getParent();
+ bool FirstGroup = true;
+ for (auto &[SrcSubReg, VGPRRC, GroupUses] :
+ getCopyGroups(SameBlockUses, DstReg, DAG.MRI, SRI, DAG.TRI)) {
+ Register GroupReg = DAG.MRI.createVirtualRegister(VGPRRC);
+ MachineInstrBuilder VGPRCopy =
+ BuildMI(*UseInst->getParent(), UseInst->getIterator(),
+ UseInst->getDebugLoc(), TII->get(TargetOpcode::COPY))
+ .addDef(GroupReg, {}, 0)
+ .addUse(DstReg, {}, SrcSubReg);
+ DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
+ LLVM_DEBUG(dbgs() << "[RewriteMFMAForm] Case2 same-BB COPY: "
+ << printReg(DstReg, SRI) << " -> "
+ << printReg(GroupReg, SRI) << " SrcSubReg="
+ << SrcSubReg << " before: " << *UseInst
+ << " inserted: " << *VGPRCopy);
+ if (FirstGroup) {
+ UpdateRegionBefore(UseInst, VGPRCopy);
+ FirstGroup = false;
+ }
+ for (MachineOperand *RU : GroupUses) {
+ RU->setReg(GroupReg);
+ if (SrcSubReg != AMDGPU::NoSubRegister)
+ RU->setSubReg(AMDGPU::NoSubRegister);
+ }
+ DstRegSet.insert(&VGPRCopy->getOperand(1));
+ }
}
// Track the register for reclassification
@@ -3245,50 +3566,53 @@ bool RewriteMFMAFormStage::rewrite(
DstRegSet.insert(Dst);
}
- // Handle the copies for dst uses.
+ // Handle the copies for cross-BB dst uses (Case 2): one COPY per
+ // (block, register, sub-reg group), before that group's earliest use.
using RUBType =
std::pair<unsigned, DenseMap<Register, SmallPtrSet<MachineOperand *, 8>>>;
for (RUBType RUBlockEntry : ReachingUseTracker) {
using RUDType = std::pair<Register, SmallPtrSet<MachineOperand *, 8>>;
for (RUDType RUDst : RUBlockEntry.second) {
- MachineOperand *OpBegin = *RUDst.second.begin();
- SlotIndex InstPt = DAG.LIS->getInstructionIndex(*OpBegin->getParent());
-
- // Find the earliest use in this block.
- for (MachineOperand *User : RUDst.second) {
- SlotIndex NewInstPt = DAG.LIS->getInstructionIndex(*User->getParent());
- if (SlotIndex::isEarlierInstr(NewInstPt, InstPt))
- InstPt = NewInstPt;
- }
-
- const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(RUDst.first);
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(DstRC);
- Register NewUseReg = DAG.MRI.createVirtualRegister(VGPRRC);
- MachineInstr *UseInst = DAG.LIS->getInstructionFromIndex(InstPt);
-
- MachineInstrBuilder VGPRCopy =
- BuildMI(*UseInst->getParent(), UseInst->getIterator(),
- UseInst->getDebugLoc(), TII->get(TargetOpcode::COPY))
- .addDef(NewUseReg, {}, 0)
- .addUse(RUDst.first, {}, 0);
- DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
-
- // If this UseInst was the first MI in the region, update the region
- // boundaries.
- auto FI = FirstMIToRegion.find(UseInst);
- if (FI != FirstMIToRegion.end()) {
- unsigned UpdateRegion = FI->second;
- DAG.Regions[UpdateRegion].first = VGPRCopy;
- FirstMIToRegion.erase(UseInst);
- }
-
- // Replace the operand for all users.
- for (MachineOperand *User : RUDst.second) {
- User->setReg(NewUseReg);
+ LLVM_DEBUG(dbgs() << "[RewriteMFMAForm] Case2: processing Dst="
+ << printReg(RUDst.first, SRI) << " in BB#"
+ << RUBlockEntry.first << " uses=" << RUDst.second.size()
+ << "\n");
+ // Sub-reg groups, each inserting at its own earliest use to minimize live
+ // range (UpdateRegionBefore is idempotent, so per-group calls are safe).
+ SmallVector<MachineOperand *, 8> UserOps(RUDst.second.begin(),
+ RUDst.second.end());
+ for (auto &[SrcSubReg, VGPRRC, GroupUses] :
+ getCopyGroups(UserOps, RUDst.first, DAG.MRI, SRI, DAG.TRI)) {
+ // Find earliest use in this group.
+ MachineInstr *UseInst =
+ (*llvm::min_element(GroupUses, [&](MachineOperand *A,
+ MachineOperand *B) {
+ return SlotIndex::isEarlierInstr(
+ DAG.LIS->getInstructionIndex(*A->getParent()),
+ DAG.LIS->getInstructionIndex(*B->getParent()));
+ }))->getParent();
+ Register GroupReg = DAG.MRI.createVirtualRegister(VGPRRC);
+ MachineInstrBuilder VGPRCopy =
+ BuildMI(*UseInst->getParent(), UseInst->getIterator(),
+ UseInst->getDebugLoc(), TII->get(TargetOpcode::COPY))
+ .addDef(GroupReg, {}, 0)
+ .addUse(RUDst.first, {}, SrcSubReg);
+ DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
+ LLVM_DEBUG({
+ dbgs() << "[RewriteMFMAForm] Case2 cross-BB group COPY: "
+ << printReg(RUDst.first, DAG.TRI) << " -> "
+ << printReg(GroupReg, DAG.TRI) << " SrcSubReg=" << SrcSubReg
+ << " BB#" << UseInst->getParent()->getNumber()
+ << " before: " << *UseInst << " inserted: " << *VGPRCopy;
+ });
+ UpdateRegionBefore(UseInst, VGPRCopy);
+ for (MachineOperand *U : GroupUses) {
+ U->setReg(GroupReg);
+ if (SrcSubReg != AMDGPU::NoSubRegister)
+ U->setSubReg(AMDGPU::NoSubRegister);
+ }
+ ReplaceMap[RUDst.first].insert(&VGPRCopy->getOperand(1));
}
-
- // Track the copy source operand for replacement.
- ReplaceMap[RUDst.first].insert(&VGPRCopy->getOperand(1));
}
}
@@ -3325,10 +3649,12 @@ bool RewriteMFMAFormStage::rewrite(
RegionPressureMap LiveInUpdater(&DAG, false);
LiveInUpdater.buildLiveRegMap();
- for (unsigned Region = 0; Region < DAG.Regions.size(); Region++)
+ // reanalyze() rebuilds live intervals for the whole function, so refresh
+ // every region; leaving others stale would skew later occupancy decisions.
+ for (unsigned Region = 0; Region < DAG.Regions.size(); Region++) {
DAG.LiveIns[Region] = LiveInUpdater.getLiveRegsForRegionIdx(Region);
-
- DAG.Pressure[RegionIdx] = DAG.getRealRegPressure(RegionIdx);
+ DAG.Pressure[Region] = DAG.getRealRegPressure(Region);
+ }
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 00fb6ac007346..44cf112c52eaf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/Rematerializer.h"
@@ -445,6 +446,21 @@ class RewriteMFMAFormStage : public GCNSchedStage {
const SIInstrInfo *TII;
const SIRegisterInfo *SRI;
+ /// Post-dominator tree over MF, computed once per stage run (rewrite() only
+ /// inserts COPYs within blocks, never changing CFG edges, so it stays valid).
+ MachinePostDominatorTree MPDT;
+
+ /// \returns true if block \p P (the candidate MFMA's own block, where Case 1
+ /// inserts the single bridge COPY) is a valid merge point for \p Src2Reg's
+ /// parallel/multi-block defs \p DefBlocks:
+ /// (A) P post-dominates every def block (the COPY reads one merged value);
+ /// (B) P dominates every use block (%MappedReg reaches every consumer).
+ /// Only the geometric (A)+(B) property; whether an MFMA reaching def is
+ /// tolerable is decided by the caller (Check 3).
+ bool isSrc2MergePoint(
+ Register Src2Reg, const MachineBasicBlock *P,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &DefBlocks) const;
+
/// Per-candidate cache of the src2 "needs VGPR" decision, computed once
/// and reused on-demand.
DenseMap<const MachineInstr *, bool> Src2NeedsVGPRCache;
@@ -471,53 +487,102 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Do the final rewrite on \p RewriteCands and insert any needed copies.
bool rewrite(ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
+ /// Per-BB summary of the src2 reaching defs that need a bridge copy.
+ struct Src2PerBBEntry {
+ MachineInstr *LastRD = nullptr;
+ LaneBitmask UnionLanes = LaneBitmask::getNone();
+ };
+
+ /// Decision variables + per-BB data for placing src2 bridge COPYs. Computed
+ /// once by computeSrc2BridgePlan and consumed by the emit helpers.
+ struct Src2BridgePlan {
+ /// Def-BBs sorted by their last-def slot index.
+ SmallVector<MachineBasicBlock *, 4> SortedBBs;
+ DenseMap<MachineBasicBlock *, Src2PerBBEntry> PerBBMap;
+ LaneBitmask Src2LiveLanes = LaneBitmask::getNone();
+ /// The src2 defs jointly cover every live lane of src2.
+ bool AllLanesCovered = false;
+ /// The def-BBs form a dominator chain AND the deepest def dominates the use
+ /// (false on a bypass path → merged full-width COPY is used instead).
+ bool IsSequential = false;
+ /// At least one reaching def writes a sub-register of src2.
+ bool HasSubregDef = false;
+ };
+
+ /// Pure analysis (no IR change): classify the src2 reaching defs
+ /// \p Src2DefsReplace of \p Src2Reg into the Src2BridgePlan decision
+ /// variables.
+ Src2BridgePlan computeSrc2BridgePlan(
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace,
+ Register Src2Reg, const MachineBasicBlock *UseMBB);
+
+ /// The bridge-COPY strategy for a candidate's src2, plus the plan it derives
+ /// from. UseMerged: true = one full-width COPY before the MFMA
+ /// (emitMergedSrc2Bridge); false = one COPY per def-BB
+ /// (emitPerDefBBSrc2Bridges). Shared by rewrite() (emit path) and
+ /// initHeuristics (COPY cost).
+ struct Src2BridgeDecision {
+ Src2BridgePlan Plan;
+ bool UseMerged = false;
+ };
+
+ /// Pure analysis (no IR change): pick the src2 bridge strategy for candidate
+ /// \p MI given its \p Src2 operand and the defs \p Src2DefsReplace to bridge.
+ Src2BridgeDecision
+ classifySrc2Bridge(MachineInstr *MI, MachineOperand *Src2,
+ const SmallSetVector<MachineInstr *, 8> &Src2DefsReplace);
+
+ /// Merged path: emit one full-width bridge COPY of \p Src2Reg into
+ /// \p MappedReg immediately before \p MI, where the LIS PHI has merged all
+ /// parallel paths.
+ void
+ emitMergedSrc2Bridge(MachineInstr *MI, Register Src2Reg, Register MappedReg,
+ function_ref<void(MachineInstr *, MachineInstrBuilder &)>
+ UpdateRegionBefore);
+
+ /// Per-def-BB path: emit one bridge COPY per def-BB after its last def,
+ /// narrowing to a sub-reg COPY (first flagged undef) when the defs jointly
+ /// cover the live range but this BB writes only part of it.
+ void emitPerDefBBSrc2Bridges(
+ Register Src2Reg, Register MappedReg, const Src2BridgePlan &Plan,
+ SmallPtrSetImpl<MachineInstr *> &ReachingDefCopies,
+ function_ref<void(MachineInstr *, MachineInstrBuilder &)>
+ UpdateRegionAfter);
+
/// \returns true if this MI is a rewrite candidate.
bool isRewriteCandidate(MachineInstr *MI) const;
/// Returns true if the src2 reaching defs \p DefIdxs of \p Src2Reg have a
/// conflict that prevents safe bridge-copy insertion after non-MAI defs.
///
- /// rewrite()'s design principle for src2: instead of reclassifying Src2Reg
- /// to AGPR directly, Case 1 creates a fresh VGPR %MappedReg and inserts a
- /// bridge COPY (Src2Reg → %MappedReg) after each non-MAI reaching def. The
- /// MFMA's src2 operand is then replaced with %MappedReg, which rewrite()
- /// subsequently reclassifies to AGPR. This preserves Src2Reg's VGPR class
- /// for non-MFMA users while giving the MFMA an AGPR-class src2.
- ///
- /// For this strategy to be correct, every bridge COPY must produce a
- /// well-defined %MappedReg at every src2 use site. Two conditions break
- /// this requirement:
- /// Check 1: if a MAI def dominates a non-MAI def, the bridge copy after the
- /// non-MAI def would need to partially update an already-AGPR register —
- /// illegal for rewrite()'s COPY model.
- /// Early-safe return: if all (MAI, non-MAI) pairs are parallel in the CFG
- /// and every MAI def is itself a candidate, the non-MAI defs can be bridged
- /// without conflict; Check 2 is skipped.
- /// Check 2: on every path from the entry to a use of \p Src2Reg, some bridge
- /// block in \p BridgeBlocks (a non-MAI def block of Src2Reg, where a bridge
- /// copy is inserted) must be crossed; otherwise %MappedReg is undefined on
- /// that path. \p BridgeBlocks is precomputed in computeExclusionSet as the
- /// union, over all candidates sharing Src2Reg, of the non-MAI reaching-def
- /// blocks of their src2 — this is what rewrite() actually inserts copies
- /// after, so a single candidate's own reaching defs are not enough.
- /// Check 3: if \p RedefPartialBlocks (non-AGPR-form partial subreg
- /// reaching-def blocks of Src2Reg) holds >=2 mutually non-dominating blocks
- /// with no single block dominating all uses, redirecting uses to %MappedReg
- /// splits the original Src2Reg live interval into disconnected components.
+ /// Case 1 bridges src2 by inserting a COPY (Src2Reg → fresh VGPR %MappedReg)
+ /// after each non-MAI reaching def and reclassifying %MappedReg to AGPR, so
+ /// %MappedReg must be well-defined at every src2 use. The checks below each
+ /// detect a way that breaks:
+ /// Check 1: a MAI def dominates a non-MAI def (bridge would partially update
+ /// an already-AGPR register).
+ /// SkipBridgeCoverage: all (MAI, non-MAI) pairs are parallel and every MAI
+ /// def is a candidate → Check 2 skipped (Check 3 still runs).
+ /// Check 2: some entry→use path crosses no bridge block in \p BridgeBlocks
+ /// (%MappedReg undefined there).
+ /// Check 3: \p RedefPartialBlocks holds >=2 mutually non-dominating blocks
+ /// with no single dominating merge block (interval fragments).
+ /// Coverage: the bridged defs don't jointly cover every live lane of Src2Reg.
bool hasSrc2BridgeConflict(
ArrayRef<SlotIndex> DefIdxs, Register Src2Reg,
+ const MachineBasicBlock *CandBlock,
const SmallPtrSetImpl<const MachineBasicBlock *> &BridgeBlocks,
- const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks)
- const;
+ const SmallPtrSetImpl<const MachineBasicBlock *> &RedefPartialBlocks,
+ const SmallSetVector<MachineInstr *, 16> &RewriteSet,
+ const DenseSet<Register> &CandSrc2Regs) const;
/// Returns true if reclassifying \p DstReg to AGPR would require bridge
- /// copies beyond rewrite()'s capability (full-register copies only).
- /// Phase 1: non-agnostic subreg writer (e.g. V_MOV) cannot write an AGPR
- /// sub-register lane → conflict. Agnostic writers (COPY/AV_MOV) lower to
- /// v_accvgpr_write and are legal but their orphan uses are checked in
- /// Phase 2. Phase 2: if an orphan use of an agnostic subreg def is
- /// non-agnostic, it cannot legally source an AGPR sub-register lane →
- /// conflict.
+ /// copies beyond rewrite()'s capability (full-register copies only). MAI
+ /// defs/uses are skipped throughout (they natively access AGPR).
+ /// Phase 1: a non-agnostic subreg writer (e.g. V_MOV) can't write an AGPR
+ /// lane → conflict; agnostic writers (COPY/AV_MOV) are deferred to Phase 2.
+ /// Phase 2: a non-agnostic use of an agnostic subreg def can't source an
+ /// AGPR lane → conflict.
bool hasDstSubregConflict(Register DstReg, MachineInstr *MFMA);
/// Transitively exclude from \p ExcludedMFMAs any rewrite candidate whose
@@ -526,10 +591,9 @@ class RewriteMFMAFormStage : public GCNSchedStage {
propagateExclusionForward(MachineInstr *Root,
SmallPtrSetImpl<MachineInstr *> &ExcludedMFMAs);
- /// Compute the set of rewrite candidates in \p RewriteSet that must be
- /// excluded from rewriting due to src2 dominance conflicts or dst subreg
- /// conflicts. Exclusion propagates forward (dst→src2 chain) and backward
- /// (MAI reaching-defs of a conflicted src2). No IR is mutated.
+ /// Compute the candidates in \p RewriteSet to exclude for src2 bridge or dst
+ /// subreg conflicts. Exclusion propagates forward (dst→src2 chain) and
+ /// backward (MAI reaching-defs of a conflicted src2). No IR is mutated.
SmallPtrSet<MachineInstr *, 16>
computeExclusionSet(const SmallSetVector<MachineInstr *, 16> &RewriteSet);
@@ -562,23 +626,40 @@ class RewriteMFMAFormStage : public GCNSchedStage {
struct MFMACopyPlan {
/// Non-AGPR-form reaching defs of src2 that need a bridge COPY (Case 1).
SmallSetVector<MachineInstr *, 8> Src2DefsNeedingCopy;
- /// Non-group reaching uses of the dst that need a copy (Case 2).
+ /// Reaching uses of the dst needing a copy, except group MAI members
+ /// (Case 2).
SmallSetVector<MachineOperand *, 8> DstUsesNeedingCopy;
/// Non-MAI reaching defs of those dst uses that need a copy (Case 3).
SmallSetVector<MachineInstr *, 8> DstUseDefsNeedingCopy;
};
- /// Compute the copy plan for MFMA candidate \p MI. \p GroupSet is the
- /// post-exclusion rewrite group and \p Src2Regs the candidate src2 registers
- /// (both used to classify AGPR-form reaching defs); \p Src2NeedsVGPR forces
- /// every src2 reaching def to be bridged.
+ /// Compute the copy plan for MFMA candidate \p MI. \p GroupSet
+ /// (post-exclusion group) and \p Src2Regs (candidate src2 regs) classify
+ /// AGPR-form reaching defs; \p Src2NeedsVGPR forces every src2 reaching def
+ /// to be bridged.
MFMACopyPlan
analyzeCopyPoints(MachineInstr *MI,
const SmallSetVector<MachineInstr *, 16> &GroupSet,
const DenseSet<Register> &Src2Regs, bool Src2NeedsVGPR);
+ /// One group of MachineOperand uses that share the same subreg of a dst
+ /// register, used by getCopyGroups to dedup and narrow Case2 bridge COPYs.
+ struct CopyGroup {
+ unsigned SrcSubReg; ///< NoSubRegister → full-width COPY
+ const TargetRegisterClass *VGPRRC;
+ SmallVector<MachineOperand *, 4> Uses;
+ };
+
+ /// Group \p Uses by subreg and compute the narrowest legal VGPR RC for each
+ /// group via AGPR-side validation. Returns one CopyGroup per distinct subreg.
+ static SmallVector<CopyGroup, 2>
+ getCopyGroups(ArrayRef<MachineOperand *> Uses, Register Reg,
+ MachineRegisterInfo &MRI, const SIRegisterInfo *SRI,
+ const TargetRegisterInfo *TRI);
+
public:
bool initGCNSchedStage() override;
+ bool shouldRevertScheduling(unsigned WavesAfter) override;
RewriteMFMAFormStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
: GCNSchedStage(StageID, DAG) {}
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-pressure-refresh.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-pressure-refresh.mir
new file mode 100644
index 0000000000000..ca9129e584b94
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-pressure-refresh.mir
@@ -0,0 +1,101 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -run-pass=machine-scheduler %s -o - | FileCheck %s
+#
+# Test: RewriteMFMAFormStage pressure refresh and shouldRevertScheduling.
+#
+# Two scheduling regions (split by SCHED_BARRIER), both under archVGPR excess.
+# Both regions' MFMAs must end up in areg form, which verifies:
+# Fix 1: rewrite() refreshes pressure for ALL regions after LIS->reanalyze(),
+# not just the current one (else other regions keep stale pressure).
+# Fix 2: shouldRevertScheduling override does not falsely revert on the
+# expected AGPR increase (only reverts if archVGPR/AGPR worsens over
+# the physical limit).
+#
+# CHECK-LABEL: name: test_pressure_refresh_multiregion
+# CHECK: bb.1.loop:
+# Region 1 (before SCHED_BARRIER): MFMAs rewritten to areg.
+# CHECK-DAG: :areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64
+# Region 0 (after SCHED_BARRIER): MFMAs also rewritten to areg.
+# CHECK-DAG: SCHED_BARRIER 0
+# CHECK-DAG: :areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64
+# No vgprcd MFMAs remain anywhere.
+# CHECK-NOT: V_MFMA_F32_16X16X32_F16_vgprcd_e64
+--- |
+ target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+ target triple = "amdgcn-amd-amdhsa"
+
+ declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #0
+
+ define amdgpu_kernel void @test_pressure_refresh_multiregion(ptr addrspace(1) %out, <8 x half> %a, <8 x half> %b, i32 %n) #1 {
+ entry:
+ br label %loop
+ loop:
+ br label %loop
+ epilogue:
+ ret void
+ }
+
+ attributes #0 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) "target-cpu"="gfx950" }
+ attributes #1 = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,1" "target-cpu"="gfx950" }
+...
+---
+name: test_pressure_refresh_multiregion
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 1
+body: |
+ bb.0.entry:
+ successors: %bb.1(0x80000000)
+
+ %0:vreg_128_align2 = IMPLICIT_DEF
+ %1:vreg_128_align2 = IMPLICIT_DEF
+ %2:av_128_align2 = IMPLICIT_DEF
+ %3:av_128_align2 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = IMPLICIT_DEF
+ %7:vreg_128_align2 = IMPLICIT_DEF
+ %8:vreg_128_align2 = IMPLICIT_DEF
+ %9:av_128_align2 = IMPLICIT_DEF
+ %10:av_128_align2 = IMPLICIT_DEF
+ ; 8 x vreg_1024 = 256 arch VGPRs live into loop: excess pressure in both regions.
+ %100:vreg_1024_align2 = IMPLICIT_DEF
+ %101:vreg_1024_align2 = IMPLICIT_DEF
+ %102:vreg_1024_align2 = IMPLICIT_DEF
+ %103:vreg_1024_align2 = IMPLICIT_DEF
+ %104:vreg_1024_align2 = IMPLICIT_DEF
+ %105:vreg_1024_align2 = IMPLICIT_DEF
+ %106:vreg_1024_align2 = IMPLICIT_DEF
+ %107:vreg_1024_align2 = IMPLICIT_DEF
+ %110:vreg_128_align2 = IMPLICIT_DEF
+ %111:vreg_128_align2 = IMPLICIT_DEF
+
+ bb.1.loop:
+ successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+ ; Region 1 (before SCHED_BARRIER): 4-level MFMA chain.
+ %0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %0, 0, 0, 0, implicit $mode, implicit $exec
+ %1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %1, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %9, %10, %0, 0, 0, 0, implicit $mode, implicit $exec
+ %5:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %9, %10, %1, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %9, %10, %6, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %100, %101, %102, %103, %104, %105, %106, %107
+ SCHED_BARRIER 0
+
+ ; Region 0 (after SCHED_BARRIER): separate 2-chain.
+ ; After rewrite() calls LIS->reanalyze(), Region 1's pressure must also be
+ ; refreshed (Fix 1). Without Fix 1, DAG.Pressure[Region 1] stays stale.
+ %120:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %110, 0, 0, 0, implicit $mode, implicit $exec
+ %121:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 %2, %3, %111, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC1 %bb.1, implicit undef $scc
+ S_BRANCH %bb.2
+
+ bb.2.epilogue:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
index 7841edcf8671c..5db6fdb266e66 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-safe-guard.mir
@@ -48,11 +48,20 @@
entry:
unreachable
}
- define void @test_src2_exclusion_fixpoint() #0 {
+ define void @test_src2_loop_partial_def_coverage_excluded() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_skip_bridge_coverage_vs_connectivity() #0 {
+ entry:
+ unreachable
+ }
+ define void @test_src2_partial_live_lanes_covered() #1 {
entry:
unreachable
}
attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+ attributes #1 = { "amdgpu-flat-work-group-size"="1,256" }
...
---
@@ -827,32 +836,95 @@ body: |
---
#
-# Exclusion is a fixpoint: excluding one candidate re-classifies another's src2.
-#
-# bb.0
-# / \
-# bb.1 bb.2 %rA.subN = V_MOV (non-AGPR-form partial def, fragments %rA)
-# \ /
-# bb.3 %resA = MFMA(.., %rA) candidate A, src2=%rA
-# / \
-# bb.4 bb.5 %acc.subN = COPY %rA.subN (partial subreg def; source = A's src2)
-# \ /
-# bb.6 %resB = MFMA(.., %acc) candidate B, src2=%acc
-#
-# A is excluded (non-dominating partial defs fragment %rA), so %rA stays VGPR and
-# the COPYs feeding %acc lose AGPR-form -- B must be excluded too. A single pass
-# misses B and rewrites it, splitting %acc's live interval; the fixpoint
-# re-classification catches B, keeping both MFMAs in vgprcd (VGPR) form.
-#
-# CHECK-LABEL: name: test_src2_exclusion_fixpoint
-# CHECK: %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
-# CHECK: %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# MFMA in a loop body reading a full-width src2 that has a single partial def
+# (sub0/sub1 only) inside the loop. The bridge defs don't cover the MFMA's live
+# src2 (sub2/sub3 never written), so the Coverage check in hasSrc2BridgeConflict
+# excludes the chain and both MFMAs stay vgprcd (VGPR).
+#
+# CHECK-LABEL: name: test_src2_loop_partial_def_coverage_excluded
+# CHECK: bb.1:
+# CHECK: %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK: %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
# CHECK-NOT: areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
-name: test_src2_exclusion_fixpoint
+name: test_src2_loop_partial_def_coverage_excluded
tracksRegLiveness: true
body: |
bb.0:
- successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ successors: %bb.1(0x80000000)
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %f0:vgpr_32 = IMPLICIT_DEF
+ %f1:vgpr_32 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x78000000), %bb.2(0x08000000)
+
+ SCHED_BARRIER 0
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ ; single def-BB, partial subreg writes only (sub0/sub1); sub2/sub3 never
+ ; written -> bridge defs don't cover the MFMA's full-width live src2 ->
+ ; excluded by the Coverage check, MFMAs stay vgprcd.
+ undef %src2.sub0:vreg_128_align2 = COPY %f0:vgpr_32
+ %src2.sub1:vreg_128_align2 = COPY %f1:vgpr_32
+
+ %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+
+ $scc = IMPLICIT_DEF
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ %tmp0:vreg_128_align2 = COPY %dst0:vreg_128_align2
+ S_ENDPGM 0, implicit %tmp0
+...
+
+---
+#
+# SkipBridgeCoverage must not skip the connectivity (Check 3) exclusion.
+#
+# bb.0
+# / \
+# bb.1 bb.5 ← bb.5: %acc = MFMA(...) (MAI reaching def, candidate)
+# / \ |
+# bb.2 bb.3 | ← bb.2/bb.3: %acc.subN = COPY %cpsrcN (agnostic, non-
+# \ / | AGPR-form partial subreg defs; source is not a candidate
+# bb.4 | src2, so isReachingDefAGPRForm is false -> RedefPartial)
+# \ /
+# bb.6 ← %res = MFMA(a, b, %acc) (src2 = %acc)
+#
+# %acc's src2 reaching defs: one candidate MFMA (bb.5, MAI, supplies its own AGPR
+# value) plus
+# two agnostic COPY partial subreg defs (bb.2/bb.3, non-MAI) that fragment %acc's
+# live interval (RedefPartialBlocks == 2, no single dominator). COPYs are chosen
+# so hasDstSubregConflict does not fire first, leaving Check 3 as the only guard.
+#
+# SkipBridgeCoverage reasons only about the MAI def and may skip Check 2, but
+# must not skip Check 3: doing so would rewrite %acc to AGPR and produce machine
+# IR that fails the verifier's def-dominates-use invariant. Check 3 excludes the
+# MFMA, which stays in vgprcd (VGPR) form.
+#
+# CHECK-LABEL: name: test_skip_bridge_coverage_vs_connectivity
+# CHECK: bb.6:
+# CHECK: %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64
+# CHECK-NOT: %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64
+name: test_skip_bridge_coverage_vs_connectivity
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
%p0:vreg_1024 = IMPLICIT_DEF
@@ -867,53 +939,115 @@ body: |
%srca:vgpr_32 = IMPLICIT_DEF
%srcb:vgpr_32 = IMPLICIT_DEF
- %cond:sgpr_32 = COPY $sgpr6
+ %accinit:vreg_128_align2 = IMPLICIT_DEF
+ %cpsrc0:vgpr_32 = IMPLICIT_DEF
+ %cpsrc1:vgpr_32 = IMPLICIT_DEF
+ %cpsrc2:vgpr_32 = IMPLICIT_DEF
+ %cpsrc3:vgpr_32 = IMPLICIT_DEF
- S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
- S_CBRANCH_SCC1 %bb.2, implicit $scc
+ S_CBRANCH_SCC1 %bb.5, implicit undef $scc
+ S_BRANCH %bb.1
bb.1:
- successors: %bb.3(0x80000000)
- undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- S_BRANCH %bb.3
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
bb.2:
- successors: %bb.3(0x80000000)
- undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- S_BRANCH %bb.3
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
bb.3:
- successors: %bb.4(0x40000000), %bb.5(0x40000000)
- %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %rA:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
-
- S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
- S_CBRANCH_SCC1 %bb.5, implicit $scc
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %cpsrc0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %cpsrc1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %cpsrc2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %cpsrc3:vgpr_32
+ S_BRANCH %bb.4
bb.4:
successors: %bb.6(0x80000000)
- undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
- %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
- %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
- %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
S_BRANCH %bb.6
bb.5:
successors: %bb.6(0x80000000)
- undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
- %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
- %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
- %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %accinit:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
S_BRANCH %bb.6
bb.6:
- %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
+
+---
+#
+# Both MFMAs read the whole %src2, but only sub0|sub2 are ever defined (bb.1
+# writes sub0, bb.2 writes sub2); sub1/sub3 are undef and so have no live
+# subrange. hasSrc2BridgeConflict's Coverage check therefore compares the bridge
+# defs (sub0|sub2) against LiveLanes = the union of %src2's subranges (sub0|sub2,
+# not the full register): they match, so no lane gap and the candidate is not
+# excluded. A skip edge (bb.0 -> bb.4) drops the per-BB bridge COPY cost, so
+# both MFMAs are rewritten to AGPR form.
+#
+# CHECK-LABEL: name: test_src2_partial_live_lanes_covered
+# CHECK: bb.1:
+# CHECK: undef %src2.sub0:vreg_128_align2 = COPY
+# CHECK: undef [[BR:%[0-9]+]].sub0:areg_128_align2 = COPY %src2.sub0
+# CHECK: bb.2:
+# CHECK: undef %src2.sub2:vreg_128_align2 = COPY
+# CHECK: [[BR]].sub2:areg_128_align2 = COPY %src2.sub2
+# CHECK: bb.3:
+# CHECK: dead %dst0:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[BR]]
+# CHECK: dead %dst1:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[BR]]
+# CHECK-NOT: vgprcd
+name: test_src2_partial_live_lanes_covered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.4(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %f0:vgpr_32 = IMPLICIT_DEF
+ %f2:vgpr_32 = IMPLICIT_DEF
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.4, implicit $scc
+ bb.1:
+ successors: %bb.2(0x80000000)
+ ; def-BB #1: writes sub0 only
+ undef %src2.sub0:vreg_128_align2 = COPY %f0:vgpr_32
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+ ; def-BB #2: writes sub2 -> union sub0|sub2, leaving sub1/sub3 undefined but
+ ; unread, so AllLanesCovered stays true.
+ %src2.sub2:vreg_128_align2 = COPY %f2:vgpr_32
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ dead %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.4:
KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-merge-point-both-rewritten.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-merge-point-both-rewritten.mir
new file mode 100644
index 0000000000000..4a4ea62b8fcf3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-merge-point-both-rewritten.mir
@@ -0,0 +1,172 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+#
+# Two chained src2 candidates, each with parallel partial defs that merge at the
+# use and neither defined by an MFMA, so both are rewritten (nothing excluded) --
+# but via two different bridge shapes.
+#
+# bb.0
+# / \
+# bb.1 bb.2 %rA.subN = V_MOV (non-AGPR-form partial def)
+# \ /
+# bb.3 %resA = MFMA(.., %rA) candidate A, src2=%rA
+# / \
+# bb.4 bb.5 %acc.subN = COPY %rA.subN (partial subreg def; source = A's src2)
+# \ /
+# bb.6 %resB = MFMA(.., %acc) candidate B, src2=%acc
+#
+# - %rA: non-AGPR-form V_MOV defs, so Case 1 emits one full-width bridge COPY
+# (%N:areg_128_align2 = COPY %rA) in bb.3 before %resA.
+# - %acc: its defs are already the subreg COPYs in bb.4/bb.5, reclassified in
+# place to areg_128_align2; %resB reads %acc directly, no extra bridge.
+
+--- |
+ define void @test_src2_merge_point_both_rewritten() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+name: test_src2_merge_point_both_rewritten
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_src2_merge_point_both_rewritten
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %cond:sgpr_32 = COPY $sgpr6
+ ; CHECK-NEXT: S_CMP_EQ_U32 %cond, 0, implicit-def $scc
+ ; CHECK-NEXT: %srca:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %srcb:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %rA
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %rA
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 %cond, 0, implicit-def $scc
+ ; CHECK-NEXT: dead %resA:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.5, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %acc.sub0:areg_128_align2 = COPY %rA.sub0
+ ; CHECK-NEXT: %acc.sub1:areg_128_align2 = COPY %rA.sub1
+ ; CHECK-NEXT: %acc.sub2:areg_128_align2 = COPY %rA.sub2
+ ; CHECK-NEXT: %acc.sub3:areg_128_align2 = COPY %rA.sub3
+ ; CHECK-NEXT: S_BRANCH %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %acc.sub0:areg_128_align2 = COPY %rA.sub0
+ ; CHECK-NEXT: %acc.sub1:areg_128_align2 = COPY %rA.sub1
+ ; CHECK-NEXT: %acc.sub2:areg_128_align2 = COPY %rA.sub2
+ ; CHECK-NEXT: %acc.sub3:areg_128_align2 = COPY %rA.sub3
+ ; CHECK-NEXT: S_BRANCH %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: dead %resB:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, %acc, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %cond:sgpr_32 = COPY $sgpr6
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+ undef %rA.sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ %rA.sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ %resA:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %rA:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CMP_EQ_U32 %cond:sgpr_32, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.4:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %rA.sub0:vreg_128_align2
+ %acc.sub1:vreg_128_align2 = COPY %rA.sub1:vreg_128_align2
+ %acc.sub2:vreg_128_align2 = COPY %rA.sub2:vreg_128_align2
+ %acc.sub3:vreg_128_align2 = COPY %rA.sub3:vreg_128_align2
+ S_BRANCH %bb.6
+
+ bb.6:
+ %resB:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mfma-plus-subreg-rewrite.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mfma-plus-subreg-rewrite.mir
new file mode 100644
index 0000000000000..7020d9d763ff4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mfma-plus-subreg-rewrite.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+#
+# Mixed MFMA-def + subreg-def src2 that IS rewritten (not excluded).
+#
+# bb.0.entry
+# / \
+# bb.1 \ skip edge bb.0 -> bb.5 (sgpr6 == 0)
+# / \ \
+# bb.2 bb.3 \
+# \ / \
+# bb.4 ------> bb.5
+#
+# %acc has two parallel, non-dominating reaching defs on the diamond arms: a
+# full MFMA def (bb.2, AGPR-form) and a subreg-wise non-MAI full def (bb.3, four
+# subreg COPYs that jointly cover the live range, VGPR); they merge at the src2
+# use in bb.4. No safe-guard excludes them, so both MFMAs rewrite to AGPR form.
+
+--- |
+ define void @mfma_plus_subreg_rewrite() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+...
+---
+name: mfma_plus_subreg_rewrite
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: mfma_plus_subreg_rewrite
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ ; CHECK-NEXT: %srca:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %srcb:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %f0:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.5, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr6, 1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.3, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %init:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %init
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X4F32_e64_:%[0-9]+]]:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %acc.sub0:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: %acc.sub1:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: %acc.sub2:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: %acc.sub3:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X4F32_e64_:%[0-9]+]]:areg_128_align2 = COPY %acc
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: dead %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[V_MFMA_F32_16X16X4F32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0.entry:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %f0:vgpr_32 = IMPLICIT_DEF
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $sgpr6
+ S_CMP_EQ_U32 $sgpr6, 1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+
+ bb.2:
+ successors: %bb.4(0x80000000)
+ %init:vreg_128_align2 = IMPLICIT_DEF
+ %acc:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %init:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ undef %acc.sub0:vreg_128_align2 = COPY %f0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %f0:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %f0:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %f0:vgpr_32
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.5
+
+ bb.5:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mixed-agpr-covered-perbb.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mixed-agpr-covered-perbb.mir
new file mode 100644
index 0000000000000..245b06e5844b0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-mixed-agpr-covered-perbb.mir
@@ -0,0 +1,144 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+
+#
+# %src2 has a mixed reaching-def set: an AGPR-form AV_MOV (filtered from the
+# bridge set) and a plain-VGPR subreg COPY. The VGPR def comes from a single
+# def-BB, so AllLanesCovered=0 -- yet the single-BB per-BB path still emits one
+# full-width COPY that reads the already-defined lanes and verifies clean. The
+# MFMA result is consumed (dst not dead), exercising the full value-semantics
+# path.
+#
+# CFG:
+#
+# bb.0
+# / \
+# bb.1 | ← sub0/sub1 = AV_MOV (AGPR-form, filtered)
+# | |
+# bb.2 | ← sub2/sub3 = COPY %f0 (plain VGPR); per-BB COPY %src2 here
+# | |
+# bb.3 | ← %dst0/%dst1 = MFMA(.., %src2)
+# | |
+# bb.4 | ← consumes %dst0/%dst1 (reached only from bb.3)
+# |
+# bb.5 ← convergence: KILL pressure regs
+#
+# The skip edge bb.0 -> bb.5 keeps the def+MFMA subgraph off half the paths, so
+# its block frequency (and thus the per-BB bridge COPY cost) rounds to zero and
+# the rewrite fires -- same mechanism as parallel-fullwidth-merged.mir.
+
+--- |
+ define void @mixed_agpr_covered_perbb() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+...
+---
+name: mixed_agpr_covered_perbb
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: mixed_agpr_covered_perbb
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ ; CHECK-NEXT: %srca:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %srcb:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %f0:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.5, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef %src2.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: %src2.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %src2.sub2:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: %src2.sub3:vreg_128_align2 = COPY %f0
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %src2
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %dst0:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %dst1:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY %dst0.sub1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY %dst0.sub0
+ ; CHECK-NEXT: %tmp0:vgpr_32 = V_ADD_U32_e32 [[COPY2]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY %dst1.sub1
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY %dst1.sub0
+ ; CHECK-NEXT: %tmp1:vgpr_32 = V_ADD_U32_e32 [[COPY4]], [[COPY3]], implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %tmp0, implicit %tmp1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %f0:vgpr_32 = IMPLICIT_DEF
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ undef %src2.sub0:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %src2.sub1:vreg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+ %src2.sub2:vreg_128_align2 = COPY %f0:vgpr_32
+ %src2.sub3:vreg_128_align2 = COPY %f0:vgpr_32
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ %dst0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %dst1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %src2:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.4:
+ ; Only reached from bb.3, so the dst uses here are always dominated by defs.
+ %tmp0:vgpr_32 = V_ADD_U32_e32 %dst0.sub0:vreg_128_align2, %dst0.sub1:vreg_128_align2, implicit $exec
+ %tmp1:vgpr_32 = V_ADD_U32_e32 %dst1.sub0:vreg_128_align2, %dst1.sub1:vreg_128_align2, implicit $exec
+ S_ENDPGM 0, implicit %tmp0, implicit %tmp1
+
+ bb.5:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fulldef-liveinterval-split.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fulldef-liveinterval-split.mir
new file mode 100644
index 0000000000000..b6707e4386f38
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fulldef-liveinterval-split.mir
@@ -0,0 +1,128 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+#
+# RewriteMFMAFormStage Case 1 (src2 bridge COPY): two parallel full-def blocks,
+# bypass edge.
+#
+# bb.0 ← fully defines %acc (sub0..sub3)
+# / \
+# | bb.1 ← fully redefines %acc (sub0..sub3)
+# \ /
+# bb.2 ← MFMA chain reads full %acc as src2
+#
+# bb.0 and bb.1 are mutually non-dominating full defs, so no def block is a
+# funnel point. But use block bb.2 is a valid merge point (post-dominates both
+# def blocks, dominates the use), so Check 3 does not exclude: Case 1 emits one
+# full-width COPY of %acc in bb.2 and rewrites the chain to AGPR form.
+
+--- |
+ define void @test_src2_parallel_fulldef_liveinterval_split() #0 { entry: unreachable }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+---
+name: test_src2_parallel_fulldef_liveinterval_split
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_src2_parallel_fulldef_liveinterval_split
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %srca:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %srcb:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %a0:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %a1:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %a2:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %a3:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef %acc.sub0:vreg_128_align2 = COPY %a0
+ ; CHECK-NEXT: %acc.sub1:vreg_128_align2 = COPY %a1
+ ; CHECK-NEXT: %acc.sub2:vreg_128_align2 = COPY %a2
+ ; CHECK-NEXT: %acc.sub3:vreg_128_align2 = COPY %a3
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %c0:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %c1:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %c2:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %c3:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef %acc.sub0:vreg_128_align2 = COPY %c0
+ ; CHECK-NEXT: %acc.sub1:vreg_128_align2 = COPY %c1
+ ; CHECK-NEXT: %acc.sub2:vreg_128_align2 = COPY %c2
+ ; CHECK-NEXT: %acc.sub3:vreg_128_align2 = COPY %c3
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.2(0x7e000000), %bb.3(0x02000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %acc
+ ; CHECK-NEXT: %res0:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %res1:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, %res0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: dead %res2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %a0:vgpr_32 = IMPLICIT_DEF
+ %a1:vgpr_32 = IMPLICIT_DEF
+ %a2:vgpr_32 = IMPLICIT_DEF
+ %a3:vgpr_32 = IMPLICIT_DEF
+ undef %acc.sub0:vreg_128_align2 = COPY %a0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %a1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %a2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %a3:vgpr_32
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ %c0:vgpr_32 = IMPLICIT_DEF
+ %c1:vgpr_32 = IMPLICIT_DEF
+ %c2:vgpr_32 = IMPLICIT_DEF
+ %c3:vgpr_32 = IMPLICIT_DEF
+ undef %acc.sub0:vreg_128_align2 = COPY %c0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %c1:vgpr_32
+ %acc.sub2:vreg_128_align2 = COPY %c2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %c3:vgpr_32
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.2(0x7e000000), %bb.3(0x02000000)
+ %res0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res0:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ bb.3:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fullwidth-merged.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fullwidth-merged.mir
new file mode 100644
index 0000000000000..4f22c6066cbc9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-parallel-fullwidth-merged.mir
@@ -0,0 +1,126 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+
+#
+# Two parallel (non-dominating) def-BBs each fully define %acc. No dominator
+# chain, so a merged single COPY before the MFMA keeps the value connected.
+#
+# CFG (skip edge bb.0 -> bb.5 lowers the MFMA-block frequency):
+#
+# bb.0
+# / \
+# bb.1 \ ← inner branch
+# / \ \
+# bb.2 bb.3 | ← bb.2/bb.3: %acc = DS_READ_B128 (parallel full defs)
+# \ / |
+# bb.4 | ← %res = MFMA(.., %acc); merged COPY %acc inserted here
+# \ /
+# bb.5 ← convergence: KILL pressure regs
+
+--- |
+ define void @parallel_fullwidth_merged() #0 {
+ entry:
+ unreachable
+ }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+...
+---
+name: parallel_fullwidth_merged
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: parallel_fullwidth_merged
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ ; CHECK-NEXT: %ptr:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.5, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr6, 1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.3, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %acc:vreg_128_align2 = DS_READ_B128_gfx9 %ptr, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %acc:vreg_128_align2 = DS_READ_B128_gfx9 %ptr, 256, 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %acc
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead %res:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 [[DEF]], [[DEF1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF2]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.5(0x40000000)
+ liveins: $vgpr0, $sgpr4_sgpr5, $sgpr6
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %ptr:vgpr_32 = IMPLICIT_DEF
+ S_CMP_EQ_U32 $sgpr6, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.5, implicit $scc
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $sgpr6
+ S_CMP_EQ_U32 $sgpr6, 1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.3, implicit $scc
+
+ bb.2:
+ successors: %bb.4(0x80000000)
+ %acc:vreg_128_align2 = DS_READ_B128_gfx9 %ptr, 0, 0, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ %acc:vreg_128_align2 = DS_READ_B128_gfx9 %ptr, 256, 0, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+ %res:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca, %srcb, %acc, 0, 0, 0, implicit $mode, implicit $exec
+ S_BRANCH %bb.5
+
+ bb.5:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-seq-bypass-dominance-gap.mir b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-seq-bypass-dominance-gap.mir
new file mode 100644
index 0000000000000..9842147fcff4e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-src2-seq-bypass-dominance-gap.mir
@@ -0,0 +1,115 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a \
+# RUN: -run-pass=machine-scheduler \
+# RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+#
+# src2 bridge (Case 1) merge-point fallback: a bypass edge breaks the deepest
+# def-BB's dominance over the use.
+#
+# CFG (bypass edge bb.0 -> bb.2 skips bb.1):
+#
+# bb.0 ← %acc.sub0_sub1 = COPY (lanes 0-1)
+# / \
+# | bb.1 ← %acc.sub2_sub3 = COPY (lanes 2-3)
+# \ /
+# bb.2 ← MFMA chain reads full %acc as src2
+#
+# def-BBs form a dominator chain (bb.0 dom bb.1), but bb.1 does not dominate the
+# use bb.2 (bypass reaches it without passing bb.1), so a per-BB bridge would
+# leave %MAPPED.sub2_sub3 undefined on the bypass path. bb.2 is a valid merge
+# point (isSrc2MergePoint), so Case 1 emits one full-width COPY before the MFMA
+# (%N = COPY %acc) instead, and the MFMAs are rewritten to AGPR form.
+
+--- |
+ define void @test_src2_seq_bypass_dominance_gap() #0 { entry: unreachable }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+---
+name: test_src2_seq_bypass_dominance_gap
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_src2_seq_bypass_dominance_gap
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %srca:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %srcb:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %init0:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %init1:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef %acc.sub0:vreg_128_align2 = COPY %init0
+ ; CHECK-NEXT: %acc.sub1:vreg_128_align2 = COPY %init1
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p8:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %init2:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %init3:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: %acc.sub2:vreg_128_align2 = COPY %init2
+ ; CHECK-NEXT: %acc.sub3:vreg_128_align2 = COPY %init3
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.2(0x7e000000), %bb.3(0x02000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %acc
+ ; CHECK-NEXT: %res0:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %res1:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, %res0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: dead %res2:areg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_e64 %srca, %srcb, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_1024 = IMPLICIT_DEF
+ %srca:vgpr_32 = IMPLICIT_DEF
+ %srcb:vgpr_32 = IMPLICIT_DEF
+ %init0:vgpr_32 = IMPLICIT_DEF
+ %init1:vgpr_32 = IMPLICIT_DEF
+ undef %acc.sub0:vreg_128_align2 = COPY %init0:vgpr_32
+ %acc.sub1:vreg_128_align2 = COPY %init1:vgpr_32
+ ; bypass edge: skip bb.1, go straight to bb.2 use
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ %init2:vgpr_32 = IMPLICIT_DEF
+ %init3:vgpr_32 = IMPLICIT_DEF
+ %acc.sub2:vreg_128_align2 = COPY %init2:vgpr_32
+ %acc.sub3:vreg_128_align2 = COPY %init3:vgpr_32
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.2(0x7e000000), %bb.3(0x02000000)
+ %res0:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %acc:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res1:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res0:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ %res2:vreg_128_align2 = nofpexcept V_MFMA_F32_16X16X4F32_vgprcd_e64 %srca:vgpr_32, %srcb:vgpr_32, %res1:vreg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+ S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+ S_BRANCH %bb.2
+
+ bb.3:
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7, %p8
+ S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
index 34cf624961f9e..6b1fb6c0b6d31 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
@@ -1794,55 +1794,38 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_ADD_U32_e32_]]
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = COPY [[DEF17]]
; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = COPY [[DEF18]]
; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = COPY [[DEF19]]
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: $scc = IMPLICIT_DEF
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit killed $scc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
- ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[COPY4]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY3]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[DEF16]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_ADD_U32_e32_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF18:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF17]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF19:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF18]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF19]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_8]]
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_10]]
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_5]]
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_7]]
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_9]]
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_4]]
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_11]]
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_6]]
- ; CHECK-NEXT: KILL [[COPY10]], [[COPY5]], [[COPY12]], [[COPY7]], [[COPY14]], [[COPY9]], [[COPY16]], [[COPY11]], [[COPY6]], [[COPY13]], [[COPY8]], [[COPY15]]
+ ; CHECK-NEXT: KILL [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_4]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_5]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_6]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_7]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_8]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_9]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_10]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_11]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
@@ -1853,20 +1836,17 @@ body: |
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub1, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: dead undef [[V_ADD_U32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub1, [[DEF15]], implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.6(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_ADD_U32_e32_]].sub0, [[DEF15]], implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_128_align2 = COPY [[COPY1]]
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_128_align2 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
; CHECK-NEXT: [[DEF20:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[COPY17]], [[COPY19]], [[COPY18]], [[V_ADD_U32_e32_1]], [[V_ADD_U32_e32_3]]
+ ; CHECK-NEXT: KILL [[DEF20]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[DEF17]], [[DEF18]], [[DEF19]], [[V_ADD_U32_e32_]], [[V_ADD_U32_e32_3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -2655,11 +2635,12 @@ body: |
; CHECK-NEXT: KILL [[COPY2]], [[COPY5]], [[COPY3]], [[COPY6]], [[COPY4]], [[COPY1]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[COPY7]].sub0, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF16]].sub1, [[COPY7]], implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY7]], [[V_ADD_U32_e32_]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY8]], [[V_ADD_U32_e32_]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -3597,7 +3578,6 @@ body: |
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[DEF11]], 0, 0, implicit $exec
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DS_READ_B128_gfx9_]]
; CHECK-NEXT: [[DEF12:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:av_128_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
@@ -3607,10 +3587,10 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[COPY]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[DS_READ_B128_gfx9_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF12]], [[DEF13]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], 4, 4, [[DEF15]].sub0, [[DEF11]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000)
@@ -3621,27 +3601,21 @@ body: |
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub0, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY1]].sub1, 256, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 256, 0, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub1, 0, 0, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[COPY2]].sub0, 256, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub1, 0, 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DEF11]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]].sub0, 256, 0, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DEF11]], [[DS_READ_B128_gfx9_]], 0, 0, implicit $exec
; CHECK-NEXT: SCHED_BARRIER 0
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]]
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]]
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[COPY3]], [[COPY5]], [[COPY4]], [[COPY6]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF14]], [[DEF16]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -5477,6 +5451,10 @@ body: |
S_ENDPGM 0
...
+# Stays in vgprcd form on cost, not safe-guard exclusion: the rewrite is legal
+# (nothing excluded) but %20's dst partially escapes to a VALU (V_ADD %20.sub0),
+# forcing extra VGPR<->AGPR bridge COPYs whose cost exceeds the spill savings, so
+# getRewriteCost > 0 and the VGPR form is preferred.
---
name: noncandidate_mfma_def_feeds_candidate_src2
tracksRegLiveness: true
@@ -5517,24 +5495,21 @@ body: |
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF16:%[0-9]+]]:vreg_128_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[DEF16]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[COPY]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3:%[0-9]+]]:areg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[DEF16]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_1]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = contract nofpexcept V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 [[DEF11]], [[DEF12]], [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_2]], 4, 4, [[DEF14]].sub0, [[DEF15]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_]]
- ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[COPY1]].sub0, [[DEF15]], implicit $exec
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64_3]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[COPY2]]
+ ; CHECK-NEXT: undef [[V_ADD_U32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_ADD_U32_e32 [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_]].sub0, [[DEF15]], implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY [[V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64_3]]
; CHECK-NEXT: SCHED_BARRIER 0
; CHECK-NEXT: [[DEF17:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY3]]
+ ; CHECK-NEXT: KILL [[DEF17]], [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[DEF8]], [[DEF9]], [[DEF10]], [[DEF13]], [[DEF16]], [[V_ADD_U32_e32_]], [[COPY]]
; CHECK-NEXT: S_NOP 0, implicit %12, implicit %13
; CHECK-NEXT: S_ENDPGM 0
bb.0:
More information about the llvm-commits
mailing list