[llvm] [AMDGPU] Per-chain MFMA->AGPR conversion (PR #217328)
Romanov Vlad via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 06:26:39 PDT 2026
https://github.com/romanovvlad updated https://github.com/llvm/llvm-project/pull/217328
>From 9e1f6b7d8a0fcbcf40b008f25f46b07080e6bab3 Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Mon, 10 Aug 2026 04:26:55 -0500
Subject: [PATCH 1/3] [AMDGPU] Per-chain MFMA->AGPR conversion
Currently RewriteMFMAFormStage evaluates all convertible MFMAs as a
single group - if the total cost is positive, the entire conversion
is rejected.
This is suboptimal when converting a subset of chains would be
profitable but converting all of them is not. Converting too many
chains can add excessive bridge copy overhead and may shift all spills
from VGPR to AGPR, while a smaller subset would bring both VGPR and
AGPR pressure within their respective limits.
To address this, the patch groups MFMAs into accumulator chains
(connected via dst->src2 edges), sorts them by length, and evaluates
converting different-sized subsets to find the best cost. Operating
at chain granularity avoids partial-chain conversions that would need
separate bridge copies per MFMA while providing minimal pressure
relief, and reduces the search space. Longer chains are preferred as
they provide more VGPR relief per bridge copy.
The -amdgpu-enable-mfma-chain-splitting option (default on) controls
this behavior. When disabled, only the all-or-nothing approach runs.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 191 ++++++++--
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 3 +-
.../CodeGen/AMDGPU/mfma-chain-splitting.mir | 338 ++++++++++++++++++
3 files changed, 509 insertions(+), 23 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 5816559fcc899..2a6fed6282a91 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -31,6 +31,7 @@
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/CodeGen/CalcSpillWeights.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
@@ -43,6 +44,7 @@
#include "llvm/MC/MCSchedule.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/ErrorHandling.h"
+#include <limits>
#define DEBUG_TYPE "machine-scheduler"
@@ -104,6 +106,12 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(true));
+static cl::opt<bool> EnableMFMAChainSplitting(
+ "amdgpu-enable-mfma-chain-splitting", cl::Hidden,
+ cl::desc("Use binary search to find optimal chain subset for MFMA->AGPR "
+ "conversion"),
+ cl::init(true));
+
namespace {
struct VGPRThresholdParser : public cl::parser<unsigned> {
@@ -1377,6 +1385,56 @@ void RewriteMFMAFormStage::findReachingUses(
}
}
+/// Identify accumulator chains: groups of MFMAs connected via dst->src2.
+static SmallVector<SmallVector<unsigned, 8>, 16>
+identifyAccChains(ArrayRef<MachineInstr *> Cands, const SIInstrInfo *TII) {
+ DenseMap<Register, unsigned> DstToCand;
+ DenseMap<Register, unsigned> Src2ToCand;
+ for (unsigned I = 0; I < Cands.size(); I++) {
+ DstToCand[Cands[I]->getOperand(0).getReg()] = I;
+ MachineOperand *Src2 =
+ TII->getNamedOperand(*Cands[I], AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ Src2ToCand[Src2->getReg()] = I;
+ }
+
+ SmallBitVector Visited(Cands.size());
+ auto WalkForward = [&](unsigned Start) {
+ SmallVector<unsigned, 8> Chain;
+ unsigned Cur = Start;
+ while (!Visited[Cur]) {
+ Visited[Cur] = true;
+ Chain.push_back(Cur);
+ auto It = Src2ToCand.find(Cands[Cur]->getOperand(0).getReg());
+ if (It == Src2ToCand.end() || Visited[It->second])
+ break;
+ Cur = It->second;
+ }
+ return Chain;
+ };
+
+ SmallVector<SmallVector<unsigned, 8>, 16> Chains;
+
+ // Collect linear chains from roots(src2 not produced by a candidate).
+ for (unsigned I = 0; I < Cands.size(); I++) {
+ MachineOperand *Src2 =
+ TII->getNamedOperand(*Cands[I], AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg() && DstToCand.count(Src2->getReg()))
+ continue;
+ Chains.push_back(WalkForward(I));
+ }
+
+ // Remaining chains should by cyclic. Collect them starting at any unvisited
+ // candidate.
+ for (unsigned I = 0; I < Cands.size(); I++) {
+ if (Visited[I])
+ continue;
+ Chains.push_back(WalkForward(I));
+ }
+
+ return Chains;
+}
+
bool RewriteMFMAFormStage::initGCNSchedStage() {
// We only need to run this pass if the architecture supports AGPRs.
// Additionally, we don't use AGPRs at occupancy levels above 1 so there
@@ -1399,20 +1457,113 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
TII = ST.getInstrInfo();
SRI = ST.getRegisterInfo();
- std::vector<std::pair<MachineInstr *, unsigned>> RewriteCands;
- DenseMap<MachineBasicBlock *, std::set<Register>> CopyForUse;
- SmallPtrSet<MachineInstr *, 8> CopyForDef;
+ // Collect all convertible MFMAs.
+ SmallVector<MachineInstr *, 32> AllCands;
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (isRewriteCandidate(&MI))
+ AllCands.push_back(&MI);
+
+ if (AllCands.empty())
+ return false;
+
+ // Identify accumulator chains and sort by size descending. We operate at
+ // chain granularity rather than individual MFMAs because:
+ // 1. It avoids converting MFMAs from different chains that would each need
+ // separate src2 and dst bridge copies (up to 4 copies for 2 MFMAs from
+ // 2 chains) while only reducing VGPR pressure for 2 instructions.
+ // 2. It reduces the search space from O(N_mfmas) to O(N_chains).
+ //
+ // Longer chains benefit more: each chain requires at most one src2 bridge
+ // copy and one dst bridge copy regardless of length, but reduces VGPR
+ // pressure proportionally to the number of chain members.
+ SmallVector<SmallVector<unsigned, 8>, 16> Chains =
+ identifyAccChains(AllCands, TII);
+ llvm::sort(Chains,
+ [](const auto &A, const auto &B) { return A.size() > B.size(); });
+
+ // Evaluate the cost of converting the first N sorted chains to AGPR form.
+ auto EvaluateProbe = [&](int N) -> int64_t {
+ SmallPtrSet<MachineInstr *, 32> ProbeFilter;
+ for (int I = 0; I < N; ++I)
+ for (unsigned Idx : Chains[I])
+ ProbeFilter.insert(AllCands[Idx]);
+
+ std::vector<std::pair<MachineInstr *, unsigned>> RC;
+ DenseMap<MachineBasicBlock *, std::set<Register>> CU;
+ SmallPtrSet<MachineInstr *, 8> CD;
+ Src2NeedsVGPRCache.clear();
+
+ if (!initHeuristics(RC, CU, CD, ProbeFilter))
+ return std::numeric_limits<int64_t>::max();
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe N=" << N << ":\n");
+ return getRewriteCost(RC, CU, CD);
+ };
+
+ // Start by evaluating all chains.
+ int64_t AllCost = EvaluateProbe(Chains.size());
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Chains.size()
+ << " Cost=" << AllCost << "\n");
+
+ int BestN = AllCost <= 0 ? Chains.size() : 0;
+
+ if (EnableMFMAChainSplitting && Chains.size() > 1) {
+ // Binary search for a good number of chains to convert. Chains are
+ // sorted by length, so we prefer converting the longest ones first as
+ // they provide the most VGPR relief per bridge copy. Converting too
+ // few chains may leave VGPRs over the limit; converting too many may
+ // push AGPRs over the limit. The search tries to find the best
+ // balance. Note: this does not guarantee a globally optimal
+ // solution as that would require evaluating all 2^K subsets of
+ // individual MFMAs. This is an approximation that works well when
+ // longer chains are more profitable. The search tracks the best
+ // cost seen across all probes to handle non-monotonicity.
+ int64_t BestCost = AllCost;
+ int Lo = 1, Hi = (int)Chains.size() - 1;
+
+ while (Lo <= Hi) {
+ int Mid = (Lo + Hi) / 2;
+ int64_t Cost = EvaluateProbe(Mid);
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Mid << " Cost=" << Cost
+ << "\n");
+
+ if (Cost < BestCost) {
+ BestCost = Cost;
+ BestN = Mid;
+ }
- if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef))
+ if (Cost <= 0)
+ Lo = Mid + 1;
+ else
+ Hi = Mid - 1;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: best N=" << BestN << "\n");
+
+ if (BestN == 0)
return false;
- int64_t Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
+ SmallPtrSet<MachineInstr *, 32> FinalFilter;
+ for (int I = 0; I < BestN; ++I)
+ for (unsigned Idx : Chains[I])
+ FinalFilter.insert(AllCands[Idx]);
- // If we haven't found the beneficial conditions, prefer the VGPR form which
- // may result in less cross RC copies.
- if (Cost > 0)
+ std::vector<std::pair<MachineInstr *, unsigned>> RewriteCands;
+ DenseMap<MachineBasicBlock *, std::set<Register>> CopyForUse;
+ SmallPtrSet<MachineInstr *, 8> CopyForDef;
+ Src2NeedsVGPRCache.clear();
+
+ if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef, FinalFilter))
return false;
+ // initHeuristics speculatively rewrites opcodes and register classes to
+ // AGPR form for pressure estimation. rewrite() expects VGPR-form opcodes
+ // (it calls getAGPRFormOp to find the AGPR variant), so reset first.
+ resetRewriteCandsToVGPR(RewriteCands);
return rewrite(RewriteCands);
}
@@ -2392,28 +2543,24 @@ bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
bool RewriteMFMAFormStage::initHeuristics(
std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
- SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
+ SmallPtrSetImpl<MachineInstr *> &CopyForDef,
+ const SmallPtrSetImpl<MachineInstr *> &RewriteSet) {
bool Changed = false;
- // Collect the candidate group, its members share AGPR-form operands
- // post-rewrite, so reaching defs feeding any member don't need bridge copy.
- SmallPtrSet<MachineInstr *, 16> RewriteSet;
+ // Collect src2 registers from the rewrite set. Members share AGPR-form
+ // operands post-rewrite, so reaching defs feeding any member don't need
+ // a bridge copy.
DenseSet<Register> CandSrc2Regs;
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
- continue;
- RewriteSet.insert(&MI);
- MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (Src2 && Src2->isReg())
- CandSrc2Regs.insert(Src2->getReg());
- }
+ for (MachineInstr *MI : RewriteSet) {
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (Src2 && Src2->isReg())
+ CandSrc2Regs.insert(Src2->getReg());
}
// Prepare for the heuristics
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
- if (!isRewriteCandidate(&MI))
+ if (!RewriteSet.contains(&MI))
continue;
int ReplacementOp = AMDGPU::getAGPRFormOp(MI.getOpcode());
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2059f4e6479ff..080d2467b63a6 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -456,7 +456,8 @@ class RewriteMFMAFormStage : public GCNSchedStage {
bool
initHeuristics(std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
- SmallPtrSetImpl<MachineInstr *> &CopyForDef);
+ SmallPtrSetImpl<MachineInstr *> &CopyForDef,
+ const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
/// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir b/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
new file mode 100644
index 0000000000000..539c1a08b4270
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
@@ -0,0 +1,338 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false %s -o - | FileCheck -check-prefix=SPLIT %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false -amdgpu-enable-mfma-chain-splitting=false %s -o - | FileCheck -check-prefix=NOSPLIT %s
+
+# Test that chain splitting finds a profitable subset of chains to convert
+# to AGPR form, vs the default mode where converting all chains is rejected.
+#
+# Without splitting: the cost of converting all 13 chains is positive
+# (copy overhead exceeds spill benefit), so conversion is fully rejected.
+#
+# With splitting: binary search finds that converting only the longest
+# chain (chain A) is profitable, producing better code.
+
+--- |
+ define amdgpu_kernel void @chain_split_profitable() #0 { ret void }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+name: chain_split_profitable
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ ; SPLIT-LABEL: name: chain_split_profitable
+ ; SPLIT: bb.0:
+ ; SPLIT-NEXT: successors: %bb.1(0x80000000)
+ ; SPLIT-NEXT: {{ $}}
+ ; SPLIT-NEXT: %p8:vreg_64_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr0:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr1:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr2:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr3:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr4:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr5:areg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %agpr7:areg_256 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initA:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %initA
+ ; SPLIT-NEXT: %initB:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initC:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initD:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initE:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initF:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initG:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initH:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initI:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initJ:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initK:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initL:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %initM:vreg_128_align2 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: SCHED_BARRIER 0
+ ; SPLIT-NEXT: S_BRANCH %bb.1
+ ; SPLIT-NEXT: {{ $}}
+ ; SPLIT-NEXT: bb.1:
+ ; SPLIT-NEXT: successors: %bb.1(0x42108421), %bb.2(0x3def7bdf)
+ ; SPLIT-NEXT: {{ $}}
+ ; SPLIT-NEXT: %a0:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %a1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
+ ; SPLIT-NEXT: %a2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
+ ; SPLIT-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ ; SPLIT-NEXT: S_BRANCH %bb.1
+ ; SPLIT-NEXT: {{ $}}
+ ; SPLIT-NEXT: bb.2:
+ ; SPLIT-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY %a2
+ ; SPLIT-NEXT: %exitA:vreg_128_align2 = COPY [[COPY1]]
+ ; SPLIT-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; SPLIT-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p8
+ ; SPLIT-NEXT: KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
+ ; SPLIT-NEXT: S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
+ ;
+ ; NOSPLIT-LABEL: name: chain_split_profitable
+ ; NOSPLIT: bb.0:
+ ; NOSPLIT-NEXT: successors: %bb.1(0x80000000)
+ ; NOSPLIT-NEXT: {{ $}}
+ ; NOSPLIT-NEXT: %p8:vreg_64_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr0:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr1:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr2:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr3:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr4:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr5:areg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %agpr7:areg_256 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initA:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initB:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initC:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initD:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initE:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initF:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initG:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initH:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initI:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initJ:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initK:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initL:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %initM:vreg_128_align2 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: SCHED_BARRIER 0
+ ; NOSPLIT-NEXT: S_BRANCH %bb.1
+ ; NOSPLIT-NEXT: {{ $}}
+ ; NOSPLIT-NEXT: bb.1:
+ ; NOSPLIT-NEXT: successors: %bb.1(0x42108421), %bb.2(0x3def7bdf)
+ ; NOSPLIT-NEXT: {{ $}}
+ ; NOSPLIT-NEXT: %a0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initA, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %a1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %a2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOSPLIT-NEXT: %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
+ ; NOSPLIT-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ ; NOSPLIT-NEXT: S_BRANCH %bb.1
+ ; NOSPLIT-NEXT: {{ $}}
+ ; NOSPLIT-NEXT: bb.2:
+ ; NOSPLIT-NEXT: %exitA:vreg_128_align2 = COPY %a2
+ ; NOSPLIT-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; NOSPLIT-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p8
+ ; NOSPLIT-NEXT: KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
+ ; NOSPLIT-NEXT: S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p8:vreg_64_align2 = IMPLICIT_DEF
+
+ %agpr0:areg_1024 = IMPLICIT_DEF
+ %agpr1:areg_1024 = IMPLICIT_DEF
+ %agpr2:areg_1024 = IMPLICIT_DEF
+ %agpr3:areg_1024 = IMPLICIT_DEF
+ %agpr4:areg_1024 = IMPLICIT_DEF
+ %agpr5:areg_1024 = IMPLICIT_DEF
+ %agpr7:areg_256 = IMPLICIT_DEF
+
+ %initA:vreg_128_align2 = IMPLICIT_DEF
+ %initB:vreg_128_align2 = IMPLICIT_DEF
+ %initC:vreg_128_align2 = IMPLICIT_DEF
+ %initD:vreg_128_align2 = IMPLICIT_DEF
+ %initE:vreg_128_align2 = IMPLICIT_DEF
+ %initF:vreg_128_align2 = IMPLICIT_DEF
+ %initG:vreg_128_align2 = IMPLICIT_DEF
+ %initH:vreg_128_align2 = IMPLICIT_DEF
+ %initI:vreg_128_align2 = IMPLICIT_DEF
+ %initJ:vreg_128_align2 = IMPLICIT_DEF
+ %initK:vreg_128_align2 = IMPLICIT_DEF
+ %initL:vreg_128_align2 = IMPLICIT_DEF
+ %initM:vreg_128_align2 = IMPLICIT_DEF
+
+ SCHED_BARRIER 0
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x10000000), %bb.2(0x0F000000)
+
+ ; Chain A (length 3).
+ %a0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initA, 0, 0, 0, implicit $mode, implicit $exec
+ %a1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
+ %a2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; Chain B (length 2): b0 → b1. b2 uses b1 as src2 but has a V_ADD
+ ; user on its dst, so b2 fails isRewriteCandidate and is not in the chain.
+ ; When chain B is converted, b1's dst becomes AGPR. b2 (not in RewriteSet)
+ ; needs a CopyForUse in the loop body — expensive at loop frequency.
+ %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
+ %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
+ %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
+
+ ; Chains C-J (length 2 each, with V_ADD user in loop).
+ %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
+ %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
+ %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
+
+ %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
+ %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
+ %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
+
+ %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
+ %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
+ %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
+
+ %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
+ %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
+ %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
+
+ %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
+ %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
+ %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
+
+ %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
+ %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
+ %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
+
+ %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
+ %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
+ %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
+
+ %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
+ %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
+ %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
+
+ %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
+ %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
+ %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
+
+ %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
+ %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
+ %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
+
+ %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
+ %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+ %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
+
+ S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ %exitA:vreg_128_align2 = COPY %a2
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p8
+ KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
+ S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
+...
>From f3e3181d5d4d303ed4ac8df24587cf8ce43ac8fe Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Fri, 21 Aug 2026 07:09:12 -0500
Subject: [PATCH 2/3] comments
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 137 ++++----
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 9 +
.../CodeGen/AMDGPU/mfma-chain-splitting.mir | 306 ++++++------------
3 files changed, 182 insertions(+), 270 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 2a6fed6282a91..5a88f742779b1 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -106,11 +106,6 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(true));
-static cl::opt<bool> EnableMFMAChainSplitting(
- "amdgpu-enable-mfma-chain-splitting", cl::Hidden,
- cl::desc("Use binary search to find optimal chain subset for MFMA->AGPR "
- "conversion"),
- cl::init(true));
namespace {
@@ -1435,6 +1430,72 @@ identifyAccChains(ArrayRef<MachineInstr *> Cands, const SIInstrInfo *TII) {
return Chains;
}
+int64_t RewriteMFMAFormStage::evaluateChainProbe(
+ int N, ArrayRef<SmallVector<unsigned, 8>> Chains,
+ ArrayRef<MachineInstr *> AllCands) {
+ SmallPtrSet<MachineInstr *, 32> ProbeFilter;
+ for (int I = 0; I < N; ++I) {
+ for (unsigned Idx : Chains[I])
+ ProbeFilter.insert(AllCands[Idx]);
+ }
+
+ std::vector<std::pair<MachineInstr *, unsigned>> RC;
+ DenseMap<MachineBasicBlock *, std::set<Register>> CU;
+ SmallPtrSet<MachineInstr *, 8> CD;
+ Src2NeedsVGPRCache.clear();
+
+ if (!initHeuristics(RC, CU, CD, ProbeFilter))
+ return std::numeric_limits<int64_t>::max();
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe N=" << N << ":\n");
+ return getRewriteCost(RC, CU, CD);
+}
+
+int RewriteMFMAFormStage::findBestChainCount(
+ ArrayRef<SmallVector<unsigned, 8>> Chains,
+ ArrayRef<MachineInstr *> AllCands) {
+ // Start by evaluating all chains.
+ int64_t AllCost = evaluateChainProbe(Chains.size(), Chains, AllCands);
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Chains.size()
+ << " Cost=" << AllCost << "\n");
+
+ int BestN = AllCost <= 0 ? Chains.size() : 0;
+
+ // Binary search for a good number of chains to convert. Chains are
+ // sorted by length, so we prefer converting the longest ones first as
+ // they provide the most VGPR relief per bridge copy. Converting too
+ // few chains may leave VGPRs over the limit; converting too many may
+ // push AGPRs over the limit. The search tries to find the best
+ // balance. Note: this does not guarantee a globally optimal
+ // solution as that would require evaluating all 2^K subsets of
+ // individual MFMAs. This is an approximation that works well when
+ // longer chains are more profitable. The search tracks the best
+ // cost seen across all probes to handle non-monotonicity.
+ int64_t BestCost = AllCost;
+ int Lo = 1, Hi = (int)Chains.size() - 1;
+
+ while (Lo <= Hi) {
+ int Mid = (Lo + Hi) / 2;
+ int64_t Cost = evaluateChainProbe(Mid, Chains, AllCands);
+
+ LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Mid << " Cost=" << Cost
+ << "\n");
+
+ if (Cost < BestCost) {
+ BestCost = Cost;
+ BestN = Mid;
+ }
+
+ if (Cost <= 0)
+ Lo = Mid + 1;
+ else
+ Hi = Mid - 1;
+ }
+
+ return BestN;
+}
+
bool RewriteMFMAFormStage::initGCNSchedStage() {
// We only need to run this pass if the architecture supports AGPRs.
// Additionally, we don't use AGPRs at occupancy levels above 1 so there
@@ -1459,10 +1520,12 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
// Collect all convertible MFMAs.
SmallVector<MachineInstr *, 32> AllCands;
- for (MachineBasicBlock &MBB : MF)
- for (MachineInstr &MI : MBB)
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
if (isRewriteCandidate(&MI))
AllCands.push_back(&MI);
+ }
+ }
if (AllCands.empty())
return false;
@@ -1482,65 +1545,7 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
llvm::sort(Chains,
[](const auto &A, const auto &B) { return A.size() > B.size(); });
- // Evaluate the cost of converting the first N sorted chains to AGPR form.
- auto EvaluateProbe = [&](int N) -> int64_t {
- SmallPtrSet<MachineInstr *, 32> ProbeFilter;
- for (int I = 0; I < N; ++I)
- for (unsigned Idx : Chains[I])
- ProbeFilter.insert(AllCands[Idx]);
-
- std::vector<std::pair<MachineInstr *, unsigned>> RC;
- DenseMap<MachineBasicBlock *, std::set<Register>> CU;
- SmallPtrSet<MachineInstr *, 8> CD;
- Src2NeedsVGPRCache.clear();
-
- if (!initHeuristics(RC, CU, CD, ProbeFilter))
- return std::numeric_limits<int64_t>::max();
-
- LLVM_DEBUG(dbgs() << "RewriteMFMA probe N=" << N << ":\n");
- return getRewriteCost(RC, CU, CD);
- };
-
- // Start by evaluating all chains.
- int64_t AllCost = EvaluateProbe(Chains.size());
-
- LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Chains.size()
- << " Cost=" << AllCost << "\n");
-
- int BestN = AllCost <= 0 ? Chains.size() : 0;
-
- if (EnableMFMAChainSplitting && Chains.size() > 1) {
- // Binary search for a good number of chains to convert. Chains are
- // sorted by length, so we prefer converting the longest ones first as
- // they provide the most VGPR relief per bridge copy. Converting too
- // few chains may leave VGPRs over the limit; converting too many may
- // push AGPRs over the limit. The search tries to find the best
- // balance. Note: this does not guarantee a globally optimal
- // solution as that would require evaluating all 2^K subsets of
- // individual MFMAs. This is an approximation that works well when
- // longer chains are more profitable. The search tracks the best
- // cost seen across all probes to handle non-monotonicity.
- int64_t BestCost = AllCost;
- int Lo = 1, Hi = (int)Chains.size() - 1;
-
- while (Lo <= Hi) {
- int Mid = (Lo + Hi) / 2;
- int64_t Cost = EvaluateProbe(Mid);
-
- LLVM_DEBUG(dbgs() << "RewriteMFMA probe: N=" << Mid << " Cost=" << Cost
- << "\n");
-
- if (Cost < BestCost) {
- BestCost = Cost;
- BestN = Mid;
- }
-
- if (Cost <= 0)
- Lo = Mid + 1;
- else
- Hi = Mid - 1;
- }
- }
+ int BestN = findBestChainCount(Chains, AllCands);
LLVM_DEBUG(dbgs() << "RewriteMFMA: best N=" << BestN << "\n");
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 080d2467b63a6..15c74074a3612 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -459,6 +459,15 @@ class RewriteMFMAFormStage : public GCNSchedStage {
SmallPtrSetImpl<MachineInstr *> &CopyForDef,
const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
+ /// Evaluate the cost of converting the first \p N chains to AGPR form.
+ int64_t evaluateChainProbe(
+ int N, ArrayRef<SmallVector<unsigned, 8>> Chains,
+ ArrayRef<MachineInstr *> AllCands);
+
+ /// Find the best number of chains to convert using binary search.
+ int findBestChainCount(ArrayRef<SmallVector<unsigned, 8>> Chains,
+ ArrayRef<MachineInstr *> AllCands);
+
/// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
/// costs, and \p RewriteCands to undo rewriting.
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir b/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
index 539c1a08b4270..3f50f706b5949 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
+++ b/llvm/test/CodeGen/AMDGPU/mfma-chain-splitting.mir
@@ -1,15 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false %s -o - | FileCheck -check-prefix=SPLIT %s
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false -amdgpu-enable-mfma-chain-splitting=false %s -o - | FileCheck -check-prefix=NOSPLIT %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false %s -o - | FileCheck %s
# Test that chain splitting finds a profitable subset of chains to convert
-# to AGPR form, vs the default mode where converting all chains is rejected.
-#
-# Without splitting: the cost of converting all 13 chains is positive
-# (copy overhead exceeds spill benefit), so conversion is fully rejected.
-#
-# With splitting: binary search finds that converting only the longest
-# chain (chain A) is profitable, producing better code.
+# to AGPR form instead of rejecting all chains when the total cost is positive.
+# The binary search finds that converting only the longest chain (chain A)
+# is profitable, while converting all 13 chains has too much copy overhead.
--- |
define amdgpu_kernel void @chain_split_profitable() #0 { ret void }
@@ -22,201 +17,104 @@ tracksRegLiveness: true
machineFunctionInfo:
isEntryFunction: true
body: |
- ; SPLIT-LABEL: name: chain_split_profitable
- ; SPLIT: bb.0:
- ; SPLIT-NEXT: successors: %bb.1(0x80000000)
- ; SPLIT-NEXT: {{ $}}
- ; SPLIT-NEXT: %p8:vreg_64_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr0:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr1:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr2:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr3:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr4:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr5:areg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %agpr7:areg_256 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initA:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %initA
- ; SPLIT-NEXT: %initB:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initC:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initD:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initE:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initF:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initG:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initH:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initI:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initJ:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initK:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initL:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %initM:vreg_128_align2 = IMPLICIT_DEF
- ; SPLIT-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: SCHED_BARRIER 0
- ; SPLIT-NEXT: S_BRANCH %bb.1
- ; SPLIT-NEXT: {{ $}}
- ; SPLIT-NEXT: bb.1:
- ; SPLIT-NEXT: successors: %bb.1(0x42108421), %bb.2(0x3def7bdf)
- ; SPLIT-NEXT: {{ $}}
- ; SPLIT-NEXT: %a0:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %a1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
- ; SPLIT-NEXT: %a2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
- ; SPLIT-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
- ; SPLIT-NEXT: S_BRANCH %bb.1
- ; SPLIT-NEXT: {{ $}}
- ; SPLIT-NEXT: bb.2:
- ; SPLIT-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY %a2
- ; SPLIT-NEXT: %exitA:vreg_128_align2 = COPY [[COPY1]]
- ; SPLIT-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; SPLIT-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p8
- ; SPLIT-NEXT: KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
- ; SPLIT-NEXT: S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
- ;
- ; NOSPLIT-LABEL: name: chain_split_profitable
- ; NOSPLIT: bb.0:
- ; NOSPLIT-NEXT: successors: %bb.1(0x80000000)
- ; NOSPLIT-NEXT: {{ $}}
- ; NOSPLIT-NEXT: %p8:vreg_64_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr0:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr1:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr2:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr3:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr4:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr5:areg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %agpr7:areg_256 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initA:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initB:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initC:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initD:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initE:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initF:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initG:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initH:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initI:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initJ:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initK:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initL:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %initM:vreg_128_align2 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: SCHED_BARRIER 0
- ; NOSPLIT-NEXT: S_BRANCH %bb.1
- ; NOSPLIT-NEXT: {{ $}}
- ; NOSPLIT-NEXT: bb.1:
- ; NOSPLIT-NEXT: successors: %bb.1(0x42108421), %bb.2(0x3def7bdf)
- ; NOSPLIT-NEXT: {{ $}}
- ; NOSPLIT-NEXT: %a0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initA, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %a1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %a2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
- ; NOSPLIT-NEXT: %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
- ; NOSPLIT-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
- ; NOSPLIT-NEXT: S_BRANCH %bb.1
- ; NOSPLIT-NEXT: {{ $}}
- ; NOSPLIT-NEXT: bb.2:
- ; NOSPLIT-NEXT: %exitA:vreg_128_align2 = COPY %a2
- ; NOSPLIT-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
- ; NOSPLIT-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p8
- ; NOSPLIT-NEXT: KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
- ; NOSPLIT-NEXT: S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
+ ; CHECK-LABEL: name: chain_split_profitable
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p8:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr0:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr1:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr2:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr3:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr4:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr5:areg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %agpr7:areg_256 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initA:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %initA
+ ; CHECK-NEXT: %initB:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initC:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initD:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initE:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initF:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initG:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initH:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initI:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initJ:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initK:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initL:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %initM:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: SCHED_BARRIER 0
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x42108421), %bb.2(0x3def7bdf)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %a0:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %a1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initB, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %b1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %c0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initC, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %c1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %c2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %c1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %d0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initD, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %d1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %d2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %d1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %e0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initE, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %e1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %e2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %e1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %f0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initF, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %f1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %f2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %f1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %g0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initG, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %g1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %g2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %g1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %h0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initH, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %h1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %h2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %h1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %i0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initI, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %i1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %i2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %i1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %j0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initJ, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %j1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %j2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %j1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %k0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initK, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %k1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %k2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %k1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %l0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initL, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %l1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %l2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %l1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %vaddB:vgpr_32 = V_ADD_U32_e32 %b2.sub0, undef %extra:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddC:vgpr_32 = V_ADD_U32_e32 %c2.sub0, undef %extra2:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddD:vgpr_32 = V_ADD_U32_e32 %d2.sub0, undef %extra3:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %initM, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %vaddE:vgpr_32 = V_ADD_U32_e32 %e2.sub0, undef %extra4:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddF:vgpr_32 = V_ADD_U32_e32 %f2.sub0, undef %extra5:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddG:vgpr_32 = V_ADD_U32_e32 %g2.sub0, undef %extra6:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %vaddH:vgpr_32 = V_ADD_U32_e32 %h2.sub0, undef %extra7:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddI:vgpr_32 = V_ADD_U32_e32 %i2.sub0, undef %extra8:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddJ:vgpr_32 = V_ADD_U32_e32 %j2.sub0, undef %extra9:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %vaddK:vgpr_32 = V_ADD_U32_e32 %k2.sub0, undef %extra10:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddL:vgpr_32 = V_ADD_U32_e32 %l2.sub0, undef %extra11:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %vaddM:vgpr_32 = V_ADD_U32_e32 %m2.sub0, undef %extra12:vgpr_32, implicit $exec
+ ; CHECK-NEXT: %a2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %a1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY %a2
+ ; CHECK-NEXT: %exitA:vreg_128_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: KILL [[DEF]], %p1, %p2, %p3, %p4, %p5, %p8
+ ; CHECK-NEXT: KILL %agpr0, %agpr1, %agpr2, %agpr3, %agpr4, %agpr5, %agpr7
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %exitA, implicit %vaddB, implicit %vaddC, implicit %vaddD, implicit %vaddE, implicit %vaddF, implicit %vaddG, implicit %vaddH, implicit %vaddI, implicit %vaddJ, implicit %vaddK, implicit %vaddL, implicit %vaddM
bb.0:
successors: %bb.1(0x80000000)
>From 6ac7cef806b989d75b0b7a51cfd704296c611913 Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Fri, 21 Aug 2026 08:26:24 -0500
Subject: [PATCH 3/3] formatting
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 -
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 5 ++---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 5a88f742779b1..da112399f99ec 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -106,7 +106,6 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(true));
-
namespace {
struct VGPRThresholdParser : public cl::parser<unsigned> {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 15c74074a3612..8253db4494e40 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -460,9 +460,8 @@ class RewriteMFMAFormStage : public GCNSchedStage {
const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
/// Evaluate the cost of converting the first \p N chains to AGPR form.
- int64_t evaluateChainProbe(
- int N, ArrayRef<SmallVector<unsigned, 8>> Chains,
- ArrayRef<MachineInstr *> AllCands);
+ int64_t evaluateChainProbe(int N, ArrayRef<SmallVector<unsigned, 8>> Chains,
+ ArrayRef<MachineInstr *> AllCands);
/// Find the best number of chains to convert using binary search.
int findBestChainCount(ArrayRef<SmallVector<unsigned, 8>> Chains,
More information about the llvm-commits
mailing list