[llvm] [AMDGPU] Move tryEffectiveStall to GCNSchedStrategy class, optionally use in tryPendingCandidate (PR #191924)
Kevin Choi via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 10:03:51 PDT 2026
https://github.com/choikwa updated https://github.com/llvm/llvm-project/pull/191924
>From ca7335942daeffc1f1c5c71319edcac0daf709ff Mon Sep 17 00:00:00 2001
From: Kevin Choi <kevin.choi at amd.com>
Date: Mon, 13 Apr 2026 20:22:27 -0500
Subject: [PATCH 1/4] [AMDGPU] Move tryEffectiveStall to GCNSchedStrategy
class, optionally use in tryPendingCandidate
This is pre-commit to allow using tryEffectiveStall in tryPendingCandidate under certain circumstances where it is beneficial to increase LDS latency.
Disabled by default.
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 38 ---------------
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 2 -
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 47 +++++++++++++++++++
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 4 ++
4 files changed, 51 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index d83f8fee2b2fe..6afd4714a2e25 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -667,44 +667,6 @@ bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
return false;
}
-bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
- SchedCandidate &TryCand,
- SchedBoundary &Zone) const {
- // Treat structural and latency stalls as a single scheduling cost for the
- // current cycle.
- struct StallCosts {
- unsigned Ready = 0;
- unsigned Structural = 0;
- unsigned Latency = 0;
- unsigned Effective = 0;
- };
-
- unsigned CurrCycle = Zone.getCurrCycle();
- auto GetStallCosts = [&](SUnit *SU) {
- unsigned ReadyCycle = Zone.isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
- StallCosts Costs;
- Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
- Costs.Structural = getStructuralStallCycles(Zone, SU);
- Costs.Latency = Zone.getLatencyStallCycles(SU);
- Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency});
- return Costs;
- };
-
- StallCosts TryCosts = GetStallCosts(TryCand.SU);
- StallCosts CandCosts = GetStallCosts(Cand.SU);
-
- LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
- dbgs() << "Effective stalls: try=" << TryCosts.Effective
- << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
- << ", lat=" << TryCosts.Latency << ") cand=" << CandCosts.Effective
- << " (ready=" << CandCosts.Ready
- << ", struct=" << CandCosts.Structural
- << ", lat=" << CandCosts.Latency << ")\n";
- });
-
- return tryLess(TryCosts.Effective, CandCosts.Effective, TryCand, Cand, Stall);
-}
-
ScheduleDAGInstrs *
llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 1684690cd829e..7334f61545c9b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -299,8 +299,6 @@ class CandidateHeuristics {
class AMDGPUCoExecSchedStrategy final : public GCNSchedStrategy {
protected:
- bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
- SchedBoundary &Zone) const;
AMDGPU::AMDGPUSchedReason LastAMDGPUReason = AMDGPU::AMDGPUSchedReason::None;
CandidateHeuristics Heurs;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index c9e7be832ec9c..f55b15717b211 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -102,6 +102,10 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(true));
+static cl::opt<bool> UseStallInPending(
+ "amdgpu-use-stall-in-pending", cl::Hidden,
+ cl::desc("Use stall based criteria when trying candidate from pending queue"), cl::init(false));
+
const unsigned ScheduleMetrics::ScaleFactor = 100;
GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
@@ -695,6 +699,44 @@ GCNSchedStageID GCNSchedStrategy::getNextStage() const {
return *std::next(CurrentStage);
}
+bool GCNSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
+ SchedCandidate &TryCand,
+ SchedBoundary &Zone) const {
+ // Treat structural and latency stalls as a single scheduling cost for the
+ // current cycle.
+ struct StallCosts {
+ unsigned Ready = 0;
+ unsigned Structural = 0;
+ unsigned Latency = 0;
+ unsigned Effective = 0;
+ };
+
+ unsigned CurrCycle = Zone.getCurrCycle();
+ auto GetStallCosts = [&](SUnit *SU) {
+ unsigned ReadyCycle = Zone.isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
+ StallCosts Costs;
+ Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
+ Costs.Structural = getStructuralStallCycles(Zone, SU);
+ Costs.Latency = Zone.getLatencyStallCycles(SU);
+ Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency});
+ return Costs;
+ };
+
+ StallCosts TryCosts = GetStallCosts(TryCand.SU);
+ StallCosts CandCosts = GetStallCosts(Cand.SU);
+
+ LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
+ dbgs() << "Effective stalls: try=" << TryCosts.Effective
+ << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
+ << ", lat=" << TryCosts.Latency << ") cand=" << CandCosts.Effective
+ << " (ready=" << CandCosts.Ready
+ << ", struct=" << CandCosts.Structural
+ << ", lat=" << CandCosts.Latency << ")\n";
+ });
+
+ return tryLess(TryCosts.Effective, CandCosts.Effective, TryCand, Cand, Stall);
+}
+
bool GCNSchedStrategy::tryPendingCandidate(SchedCandidate &Cand,
SchedCandidate &TryCand,
SchedBoundary *Zone) const {
@@ -723,6 +765,11 @@ bool GCNSchedStrategy::tryPendingCandidate(SchedCandidate &Cand,
bool SameBoundary = Zone != nullptr;
if (SameBoundary) {
+ // Compare candidates by the stall they would introduce if
+ // scheduled in the current cycle.
+ if (UseStallInPending && tryEffectiveStall(Cand, TryCand, *Zone))
+ return TryCand.Reason != NoCand;
+
TryCand.initResourceDelta(DAG, SchedModel);
if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
TryCand, Cand, ResourceReduce))
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2cc9e81a65191..48054fde76d1d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -73,6 +73,10 @@ class GCNSchedStrategy : public GenericScheduler {
bool tryPendingCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
SchedBoundary *Zone) const;
+
+ bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
+ SchedBoundary &Zone) const;
+
void printCandidateDecision(const SchedCandidate &Current,
const SchedCandidate &Preferred);
>From f7a55777bd42999a2f3c6effdb3b7844eb785b92 Mon Sep 17 00:00:00 2001
From: Kevin Choi <kevin.choi at amd.com>
Date: Mon, 13 Apr 2026 23:28:48 -0500
Subject: [PATCH 2/4] add ds_read_b128 & mfma MIR test
---
.../test/CodeGen/AMDGPU/ds_read_b128_mfma.mir | 383 ++++++++++++++++++
1 file changed, 383 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir b/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
new file mode 100644
index 0000000000000..ff671a4877d4c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
@@ -0,0 +1,383 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+# MIR fragments for gfx950-style DS_READ_B128_gfx9 -> V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64.
+# Pattern: all DS_READ_B128_gfx9 at top, all MFMA consumers at bottom.
+# REG_SEQUENCE# are DS_READ Defs that are consumed by MFMA's first Use arguments.
+# Better scheduling should load REG_SEQUENCE#s some cycles ahead of MFMA users.
+#
+
+---
+name: ds_read_mfma_2_clustered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $exec
+ ; GCN-LABEL: name: ds_read_mfma_2_clustered
+ ; GCN: liveins: $exec
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; GCN-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; GCN-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; GCN-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; GCN-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; GCN-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; GCN-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY15]], %subreg.sub4, [[COPY14]], %subreg.sub5, [[COPY13]], %subreg.sub6, [[COPY12]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF2]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; GCN-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; GCN-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; GCN-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; GCN-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; GCN-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; GCN-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; GCN-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; GCN-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %50:vreg_512_align2 = IMPLICIT_DEF
+ %51:vreg_512_align2 = IMPLICIT_DEF
+
+ %10:av_128_align2 = DS_READ_B128_gfx9 %0, 0, 0, implicit $exec
+ %11:av_32 = COPY %10.sub3
+ %12:av_32 = COPY %10.sub2
+ %13:av_32 = COPY %10.sub1
+ %14:av_32 = COPY %10.sub0
+ %15:av_128_align2 = DS_READ_B128_gfx9 %0, 8192, 0, implicit $exec
+ %16:av_32 = COPY %15.sub3
+ %17:av_32 = COPY %15.sub2
+ %18:av_32 = COPY %15.sub1
+ %19:av_32 = COPY %15.sub0
+
+ %20:av_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec
+ %21:av_32 = COPY %20.sub3
+ %22:av_32 = COPY %20.sub2
+ %23:av_32 = COPY %20.sub1
+ %24:av_32 = COPY %20.sub0
+ %25:av_128_align2 = DS_READ_B128_gfx9 %1, 8192, 0, implicit $exec
+ %26:av_32 = COPY %25.sub3
+ %27:av_32 = COPY %25.sub2
+ %28:av_32 = COPY %25.sub1
+ %29:av_32 = COPY %25.sub0
+
+ %30:av_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %31:av_32 = COPY %30.sub3
+ %32:av_32 = COPY %30.sub2
+ %33:av_32 = COPY %30.sub1
+ %34:av_32 = COPY %30.sub0
+ %35:av_128_align2 = DS_READ_B128_gfx9 %2, 8192, 0, implicit $exec
+ %36:av_32 = COPY %35.sub3
+ %37:av_32 = COPY %35.sub2
+ %38:av_32 = COPY %35.sub1
+ %39:av_32 = COPY %35.sub0
+
+ %40:av_256_align2 = REG_SEQUENCE killed %14, %subreg.sub0, killed %13, %subreg.sub1, killed %12, %subreg.sub2, killed %11, %subreg.sub3, killed %19, %subreg.sub4, killed %18, %subreg.sub5, killed %17, %subreg.sub6, killed %16, %subreg.sub7
+ %41:av_256_align2 = REG_SEQUENCE killed %24, %subreg.sub0, killed %23, %subreg.sub1, killed %22, %subreg.sub2, killed %21, %subreg.sub3, killed %29, %subreg.sub4, killed %28, %subreg.sub5, killed %27, %subreg.sub6, killed %26, %subreg.sub7
+ %42:av_256_align2 = REG_SEQUENCE killed %34, %subreg.sub0, killed %33, %subreg.sub1, killed %32, %subreg.sub2, killed %31, %subreg.sub3, killed %39, %subreg.sub4, killed %38, %subreg.sub5, killed %37, %subreg.sub6, killed %36, %subreg.sub7
+
+ %52:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %41, %40, %50, 0, 0, implicit $mode, implicit $exec
+ %53:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %42, %40, %51, 0, 0, implicit $mode, implicit $exec
+...
+---
+name: ds_read_mfma_3_clustered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $exec
+
+ ; GCN-LABEL: name: ds_read_mfma_3_clustered
+ ; GCN: liveins: $exec
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; GCN-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; GCN-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; GCN-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; GCN-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; GCN-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; GCN-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY15]], %subreg.sub4, [[COPY14]], %subreg.sub5, [[COPY13]], %subreg.sub6, [[COPY12]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF2]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; GCN-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; GCN-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; GCN-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; GCN-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; GCN-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; GCN-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; GCN-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; GCN-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_6:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_7:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY24:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub3
+ ; GCN-NEXT: [[COPY25:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub2
+ ; GCN-NEXT: [[COPY26:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub1
+ ; GCN-NEXT: [[COPY27:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub0
+ ; GCN-NEXT: [[COPY28:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub3
+ ; GCN-NEXT: [[COPY29:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub2
+ ; GCN-NEXT: [[COPY30:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub1
+ ; GCN-NEXT: [[COPY31:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub0
+ ; GCN-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY27]], %subreg.sub0, [[COPY26]], %subreg.sub1, [[COPY25]], %subreg.sub2, [[COPY24]], %subreg.sub3, [[COPY31]], %subreg.sub4, [[COPY30]], %subreg.sub5, [[COPY29]], %subreg.sub6, [[COPY28]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_2:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]], [[DEF6]], 0, 0, implicit $mode, implicit $exec
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ %60:vreg_512_align2 = IMPLICIT_DEF
+ %61:vreg_512_align2 = IMPLICIT_DEF
+ %62:vreg_512_align2 = IMPLICIT_DEF
+
+ %10:av_128_align2 = DS_READ_B128_gfx9 %0, 0, 0, implicit $exec
+ %11:av_32 = COPY %10.sub3
+ %12:av_32 = COPY %10.sub2
+ %13:av_32 = COPY %10.sub1
+ %14:av_32 = COPY %10.sub0
+ %15:av_128_align2 = DS_READ_B128_gfx9 %0, 8192, 0, implicit $exec
+ %16:av_32 = COPY %15.sub3
+ %17:av_32 = COPY %15.sub2
+ %18:av_32 = COPY %15.sub1
+ %19:av_32 = COPY %15.sub0
+
+ %20:av_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec
+ %21:av_32 = COPY %20.sub3
+ %22:av_32 = COPY %20.sub2
+ %23:av_32 = COPY %20.sub1
+ %24:av_32 = COPY %20.sub0
+ %25:av_128_align2 = DS_READ_B128_gfx9 %1, 8192, 0, implicit $exec
+ %26:av_32 = COPY %25.sub3
+ %27:av_32 = COPY %25.sub2
+ %28:av_32 = COPY %25.sub1
+ %29:av_32 = COPY %25.sub0
+
+ %30:av_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %31:av_32 = COPY %30.sub3
+ %32:av_32 = COPY %30.sub2
+ %33:av_32 = COPY %30.sub1
+ %34:av_32 = COPY %30.sub0
+ %35:av_128_align2 = DS_READ_B128_gfx9 %2, 8192, 0, implicit $exec
+ %36:av_32 = COPY %35.sub3
+ %37:av_32 = COPY %35.sub2
+ %38:av_32 = COPY %35.sub1
+ %39:av_32 = COPY %35.sub0
+
+ %40:av_128_align2 = DS_READ_B128_gfx9 %3, 0, 0, implicit $exec
+ %41:av_32 = COPY %40.sub3
+ %42:av_32 = COPY %40.sub2
+ %43:av_32 = COPY %40.sub1
+ %44:av_32 = COPY %40.sub0
+ %45:av_128_align2 = DS_READ_B128_gfx9 %3, 8192, 0, implicit $exec
+ %46:av_32 = COPY %45.sub3
+ %47:av_32 = COPY %45.sub2
+ %48:av_32 = COPY %45.sub1
+ %49:av_32 = COPY %45.sub0
+
+ %50:av_256_align2 = REG_SEQUENCE killed %14, %subreg.sub0, killed %13, %subreg.sub1, killed %12, %subreg.sub2, killed %11, %subreg.sub3, killed %19, %subreg.sub4, killed %18, %subreg.sub5, killed %17, %subreg.sub6, killed %16, %subreg.sub7
+ %51:av_256_align2 = REG_SEQUENCE killed %24, %subreg.sub0, killed %23, %subreg.sub1, killed %22, %subreg.sub2, killed %21, %subreg.sub3, killed %29, %subreg.sub4, killed %28, %subreg.sub5, killed %27, %subreg.sub6, killed %26, %subreg.sub7
+ %52:av_256_align2 = REG_SEQUENCE killed %34, %subreg.sub0, killed %33, %subreg.sub1, killed %32, %subreg.sub2, killed %31, %subreg.sub3, killed %39, %subreg.sub4, killed %38, %subreg.sub5, killed %37, %subreg.sub6, killed %36, %subreg.sub7
+ %53:av_256_align2 = REG_SEQUENCE killed %44, %subreg.sub0, killed %43, %subreg.sub1, killed %42, %subreg.sub2, killed %41, %subreg.sub3, killed %49, %subreg.sub4, killed %48, %subreg.sub5, killed %47, %subreg.sub6, killed %46, %subreg.sub7
+
+ %63:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %51, %50, %60, 0, 0, implicit $mode, implicit $exec
+ %64:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %52, %50, %61, 0, 0, implicit $mode, implicit $exec
+ %65:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %53, %50, %62, 0, 0, implicit $mode, implicit $exec
+...
+---
+name: ds_read_mfma_4_clustered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $exec
+
+ ; GCN-LABEL: name: ds_read_mfma_4_clustered
+ ; GCN: liveins: $exec
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; GCN-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; GCN-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; GCN-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; GCN-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; GCN-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; GCN-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY15]], %subreg.sub4, [[COPY14]], %subreg.sub5, [[COPY13]], %subreg.sub6, [[COPY12]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF2]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; GCN-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; GCN-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; GCN-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; GCN-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; GCN-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; GCN-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; GCN-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; GCN-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_6:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_7:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY24:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub3
+ ; GCN-NEXT: [[COPY25:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub2
+ ; GCN-NEXT: [[COPY26:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub1
+ ; GCN-NEXT: [[COPY27:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub0
+ ; GCN-NEXT: [[COPY28:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub3
+ ; GCN-NEXT: [[COPY29:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub2
+ ; GCN-NEXT: [[COPY30:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub1
+ ; GCN-NEXT: [[COPY31:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub0
+ ; GCN-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY27]], %subreg.sub0, [[COPY26]], %subreg.sub1, [[COPY25]], %subreg.sub2, [[COPY24]], %subreg.sub3, [[COPY31]], %subreg.sub4, [[COPY30]], %subreg.sub5, [[COPY29]], %subreg.sub6, [[COPY28]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_2:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]], [[DEF6]], 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_8:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 0, 0, implicit $exec
+ ; GCN-NEXT: [[DS_READ_B128_gfx9_9:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 8192, 0, implicit $exec
+ ; GCN-NEXT: [[COPY32:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub3
+ ; GCN-NEXT: [[COPY33:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub2
+ ; GCN-NEXT: [[COPY34:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub1
+ ; GCN-NEXT: [[COPY35:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub0
+ ; GCN-NEXT: [[COPY36:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub3
+ ; GCN-NEXT: [[COPY37:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub2
+ ; GCN-NEXT: [[COPY38:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub1
+ ; GCN-NEXT: [[COPY39:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub0
+ ; GCN-NEXT: [[DEF8:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; GCN-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY35]], %subreg.sub0, [[COPY34]], %subreg.sub1, [[COPY33]], %subreg.sub2, [[COPY32]], %subreg.sub3, [[COPY39]], %subreg.sub4, [[COPY38]], %subreg.sub5, [[COPY37]], %subreg.sub6, [[COPY36]], %subreg.sub7
+ ; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_3:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]], [[DEF8]], 0, 0, implicit $mode, implicit $exec
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %70:vreg_512_align2 = IMPLICIT_DEF
+ %71:vreg_512_align2 = IMPLICIT_DEF
+ %72:vreg_512_align2 = IMPLICIT_DEF
+ %73:vreg_512_align2 = IMPLICIT_DEF
+
+ %10:av_128_align2 = DS_READ_B128_gfx9 %0, 0, 0, implicit $exec
+ %11:av_32 = COPY %10.sub3
+ %12:av_32 = COPY %10.sub2
+ %13:av_32 = COPY %10.sub1
+ %14:av_32 = COPY %10.sub0
+ %15:av_128_align2 = DS_READ_B128_gfx9 %0, 8192, 0, implicit $exec
+ %16:av_32 = COPY %15.sub3
+ %17:av_32 = COPY %15.sub2
+ %18:av_32 = COPY %15.sub1
+ %19:av_32 = COPY %15.sub0
+
+ %20:av_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec
+ %21:av_32 = COPY %20.sub3
+ %22:av_32 = COPY %20.sub2
+ %23:av_32 = COPY %20.sub1
+ %24:av_32 = COPY %20.sub0
+ %25:av_128_align2 = DS_READ_B128_gfx9 %1, 8192, 0, implicit $exec
+ %26:av_32 = COPY %25.sub3
+ %27:av_32 = COPY %25.sub2
+ %28:av_32 = COPY %25.sub1
+ %29:av_32 = COPY %25.sub0
+
+ %30:av_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %31:av_32 = COPY %30.sub3
+ %32:av_32 = COPY %30.sub2
+ %33:av_32 = COPY %30.sub1
+ %34:av_32 = COPY %30.sub0
+ %35:av_128_align2 = DS_READ_B128_gfx9 %2, 8192, 0, implicit $exec
+ %36:av_32 = COPY %35.sub3
+ %37:av_32 = COPY %35.sub2
+ %38:av_32 = COPY %35.sub1
+ %39:av_32 = COPY %35.sub0
+
+ %40:av_128_align2 = DS_READ_B128_gfx9 %3, 0, 0, implicit $exec
+ %41:av_32 = COPY %40.sub3
+ %42:av_32 = COPY %40.sub2
+ %43:av_32 = COPY %40.sub1
+ %44:av_32 = COPY %40.sub0
+ %45:av_128_align2 = DS_READ_B128_gfx9 %3, 8192, 0, implicit $exec
+ %46:av_32 = COPY %45.sub3
+ %47:av_32 = COPY %45.sub2
+ %48:av_32 = COPY %45.sub1
+ %49:av_32 = COPY %45.sub0
+
+ %50:av_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec
+ %51:av_32 = COPY %50.sub3
+ %52:av_32 = COPY %50.sub2
+ %53:av_32 = COPY %50.sub1
+ %54:av_32 = COPY %50.sub0
+ %55:av_128_align2 = DS_READ_B128_gfx9 %4, 8192, 0, implicit $exec
+ %56:av_32 = COPY %55.sub3
+ %57:av_32 = COPY %55.sub2
+ %58:av_32 = COPY %55.sub1
+ %59:av_32 = COPY %55.sub0
+
+ %60:av_256_align2 = REG_SEQUENCE killed %14, %subreg.sub0, killed %13, %subreg.sub1, killed %12, %subreg.sub2, killed %11, %subreg.sub3, killed %19, %subreg.sub4, killed %18, %subreg.sub5, killed %17, %subreg.sub6, killed %16, %subreg.sub7
+ %61:av_256_align2 = REG_SEQUENCE killed %24, %subreg.sub0, killed %23, %subreg.sub1, killed %22, %subreg.sub2, killed %21, %subreg.sub3, killed %29, %subreg.sub4, killed %28, %subreg.sub5, killed %27, %subreg.sub6, killed %26, %subreg.sub7
+ %62:av_256_align2 = REG_SEQUENCE killed %34, %subreg.sub0, killed %33, %subreg.sub1, killed %32, %subreg.sub2, killed %31, %subreg.sub3, killed %39, %subreg.sub4, killed %38, %subreg.sub5, killed %37, %subreg.sub6, killed %36, %subreg.sub7
+ %63:av_256_align2 = REG_SEQUENCE killed %44, %subreg.sub0, killed %43, %subreg.sub1, killed %42, %subreg.sub2, killed %41, %subreg.sub3, killed %49, %subreg.sub4, killed %48, %subreg.sub5, killed %47, %subreg.sub6, killed %46, %subreg.sub7
+ %64:av_256_align2 = REG_SEQUENCE killed %54, %subreg.sub0, killed %53, %subreg.sub1, killed %52, %subreg.sub2, killed %51, %subreg.sub3, killed %59, %subreg.sub4, killed %58, %subreg.sub5, killed %57, %subreg.sub6, killed %56, %subreg.sub7
+
+ %74:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %61, %60, %70, 0, 0, implicit $mode, implicit $exec
+ %75:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %62, %60, %71, 0, 0, implicit $mode, implicit $exec
+ %76:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %63, %60, %72, 0, 0, implicit $mode, implicit $exec
+ %77:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 %64, %60, %73, 0, 0, implicit $mode, implicit $exec
+...
>From a4add2fabc95155dd572307c91bef01d3e9f827d Mon Sep 17 00:00:00 2001
From: Kevin Choi <kevin.choi at amd.com>
Date: Mon, 13 Apr 2026 23:41:58 -0500
Subject: [PATCH 3/4] formatting
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 4 +++-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 1 -
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index f55b15717b211..7c18db9cb47ef 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -104,7 +104,9 @@ static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
static cl::opt<bool> UseStallInPending(
"amdgpu-use-stall-in-pending", cl::Hidden,
- cl::desc("Use stall based criteria when trying candidate from pending queue"), cl::init(false));
+ cl::desc(
+ "Use stall based criteria when trying candidate from pending queue"),
+ cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 48054fde76d1d..79ebf088eaade 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -73,7 +73,6 @@ class GCNSchedStrategy : public GenericScheduler {
bool tryPendingCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
SchedBoundary *Zone) const;
-
bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
SchedBoundary &Zone) const;
>From dd93db0f745cb82c4b5c8d2016c9eb444fcb1da3 Mon Sep 17 00:00:00 2001
From: Kevin Choi <kevin.choi at amd.com>
Date: Tue, 14 Apr 2026 12:03:17 -0500
Subject: [PATCH 4/4] Test using the option
---
.../test/CodeGen/AMDGPU/ds_read_b128_mfma.mir | 175 ++++++++++++++++++
1 file changed, 175 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir b/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
index ff671a4877d4c..7b083a20cb0bc 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds_read_b128_mfma.mir
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -amdgpu-use-stall-in-pending -run-pass=machine-scheduler -verify-machineinstrs -o - %s | FileCheck -check-prefix=USESTALL %s
# MIR fragments for gfx950-style DS_READ_B128_gfx9 -> V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64.
# Pattern: all DS_READ_B128_gfx9 at top, all MFMA consumers at bottom.
@@ -56,6 +57,50 @@ body: |
; GCN-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ ;
+ ; USESTALL-LABEL: name: ds_read_mfma_2_clustered
+ ; USESTALL: liveins: $exec
+ ; USESTALL-NEXT: {{ $}}
+ ; USESTALL-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF3:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; USESTALL-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; USESTALL-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; USESTALL-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; USESTALL-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; USESTALL-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; USESTALL-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; USESTALL-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; USESTALL-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; USESTALL-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; USESTALL-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; USESTALL-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; USESTALL-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; USESTALL-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; USESTALL-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; USESTALL-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; USESTALL-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; USESTALL-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; USESTALL-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; USESTALL-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; USESTALL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; USESTALL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY15]], %subreg.sub4, [[COPY14]], %subreg.sub5, [[COPY13]], %subreg.sub6, [[COPY12]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF3]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
%2:vgpr_32 = IMPLICIT_DEF
@@ -166,6 +211,64 @@ body: |
; GCN-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY27]], %subreg.sub0, [[COPY26]], %subreg.sub1, [[COPY25]], %subreg.sub2, [[COPY24]], %subreg.sub3, [[COPY31]], %subreg.sub4, [[COPY30]], %subreg.sub5, [[COPY29]], %subreg.sub6, [[COPY28]], %subreg.sub7
; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_2:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]], [[DEF6]], 0, 0, implicit $mode, implicit $exec
+ ;
+ ; USESTALL-LABEL: name: ds_read_mfma_3_clustered
+ ; USESTALL: liveins: $exec
+ ; USESTALL-NEXT: {{ $}}
+ ; USESTALL-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF5:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; USESTALL-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; USESTALL-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; USESTALL-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; USESTALL-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; USESTALL-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; USESTALL-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; USESTALL-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; USESTALL-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; USESTALL-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; USESTALL-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; USESTALL-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; USESTALL-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; USESTALL-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; USESTALL-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; USESTALL-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; USESTALL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; USESTALL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY12]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY10]], %subreg.sub3, [[COPY17]], %subreg.sub4, [[COPY16]], %subreg.sub5, [[COPY15]], %subreg.sub6, [[COPY14]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF5]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; USESTALL-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_6:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; USESTALL-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; USESTALL-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_7:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY24:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub3
+ ; USESTALL-NEXT: [[COPY25:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub2
+ ; USESTALL-NEXT: [[COPY26:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub1
+ ; USESTALL-NEXT: [[COPY27:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub0
+ ; USESTALL-NEXT: [[COPY28:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub3
+ ; USESTALL-NEXT: [[COPY29:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub2
+ ; USESTALL-NEXT: [[COPY30:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub1
+ ; USESTALL-NEXT: [[COPY31:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub0
+ ; USESTALL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY19]], %subreg.sub0, [[COPY18]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY23]], %subreg.sub4, [[COPY22]], %subreg.sub5, [[COPY21]], %subreg.sub6, [[COPY20]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY27]], %subreg.sub0, [[COPY26]], %subreg.sub1, [[COPY25]], %subreg.sub2, [[COPY24]], %subreg.sub3, [[COPY31]], %subreg.sub4, [[COPY30]], %subreg.sub5, [[COPY29]], %subreg.sub6, [[COPY28]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_2:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]], [[DEF6]], 0, 0, implicit $mode, implicit $exec
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
%2:vgpr_32 = IMPLICIT_DEF
@@ -305,6 +408,78 @@ body: |
; GCN-NEXT: [[DEF8:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; GCN-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY35]], %subreg.sub0, [[COPY34]], %subreg.sub1, [[COPY33]], %subreg.sub2, [[COPY32]], %subreg.sub3, [[COPY39]], %subreg.sub4, [[COPY38]], %subreg.sub5, [[COPY37]], %subreg.sub6, [[COPY36]], %subreg.sub7
; GCN-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_3:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]], [[DEF8]], 0, 0, implicit $mode, implicit $exec
+ ;
+ ; USESTALL-LABEL: name: ds_read_mfma_4_clustered
+ ; USESTALL: liveins: $exec
+ ; USESTALL-NEXT: {{ $}}
+ ; USESTALL-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub3
+ ; USESTALL-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub2
+ ; USESTALL-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub1
+ ; USESTALL-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_]].sub0
+ ; USESTALL-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub3
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF1]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY5:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub2
+ ; USESTALL-NEXT: [[COPY6:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub1
+ ; USESTALL-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_1]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF2]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub3
+ ; USESTALL-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub2
+ ; USESTALL-NEXT: [[COPY10:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub1
+ ; USESTALL-NEXT: [[COPY11:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_4]].sub0
+ ; USESTALL-NEXT: [[COPY12:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub3
+ ; USESTALL-NEXT: [[COPY13:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub2
+ ; USESTALL-NEXT: [[COPY14:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub1
+ ; USESTALL-NEXT: [[COPY15:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_5]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_6:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3, [[COPY7]], %subreg.sub4, [[COPY6]], %subreg.sub5, [[COPY5]], %subreg.sub6, [[COPY4]], %subreg.sub7
+ ; USESTALL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY9]], %subreg.sub2, [[COPY8]], %subreg.sub3, [[COPY15]], %subreg.sub4, [[COPY14]], %subreg.sub5, [[COPY13]], %subreg.sub6, [[COPY12]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[DEF4]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[DEF7:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[COPY16:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub3
+ ; USESTALL-NEXT: [[COPY17:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub2
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_7:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY18:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub3
+ ; USESTALL-NEXT: [[COPY19:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub2
+ ; USESTALL-NEXT: [[COPY20:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub1
+ ; USESTALL-NEXT: [[COPY21:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_6]].sub0
+ ; USESTALL-NEXT: [[COPY22:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub3
+ ; USESTALL-NEXT: [[COPY23:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub2
+ ; USESTALL-NEXT: [[COPY24:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub1
+ ; USESTALL-NEXT: [[COPY25:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_7]].sub0
+ ; USESTALL-NEXT: [[COPY26:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub1
+ ; USESTALL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY21]], %subreg.sub0, [[COPY20]], %subreg.sub1, [[COPY19]], %subreg.sub2, [[COPY18]], %subreg.sub3, [[COPY25]], %subreg.sub4, [[COPY24]], %subreg.sub5, [[COPY23]], %subreg.sub6, [[COPY22]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_1:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[DEF7]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[COPY27:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_2]].sub0
+ ; USESTALL-NEXT: [[COPY28:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub3
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_8:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY29:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub2
+ ; USESTALL-NEXT: [[COPY30:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub1
+ ; USESTALL-NEXT: [[COPY31:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_3]].sub0
+ ; USESTALL-NEXT: [[DS_READ_B128_gfx9_9:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF5]], 8192, 0, implicit $exec
+ ; USESTALL-NEXT: [[COPY32:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub3
+ ; USESTALL-NEXT: [[COPY33:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub2
+ ; USESTALL-NEXT: [[COPY34:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub1
+ ; USESTALL-NEXT: [[COPY35:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_8]].sub0
+ ; USESTALL-NEXT: [[COPY36:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub3
+ ; USESTALL-NEXT: [[COPY37:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub2
+ ; USESTALL-NEXT: [[COPY38:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub1
+ ; USESTALL-NEXT: [[COPY39:%[0-9]+]]:av_32 = COPY [[DS_READ_B128_gfx9_9]].sub0
+ ; USESTALL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY27]], %subreg.sub0, [[COPY26]], %subreg.sub1, [[COPY17]], %subreg.sub2, [[COPY16]], %subreg.sub3, [[COPY31]], %subreg.sub4, [[COPY30]], %subreg.sub5, [[COPY29]], %subreg.sub6, [[COPY28]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_2:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]], [[DEF6]], 0, 0, implicit $mode, implicit $exec
+ ; USESTALL-NEXT: [[DEF8:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; USESTALL-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:av_256_align2 = REG_SEQUENCE [[COPY35]], %subreg.sub0, [[COPY34]], %subreg.sub1, [[COPY33]], %subreg.sub2, [[COPY32]], %subreg.sub3, [[COPY39]], %subreg.sub4, [[COPY38]], %subreg.sub5, [[COPY37]], %subreg.sub6, [[COPY36]], %subreg.sub7
+ ; USESTALL-NEXT: dead [[V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64_3:%[0-9]+]]:vreg_512_align2 = V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]], [[DEF8]], 0, 0, implicit $mode, implicit $exec
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
%2:vgpr_32 = IMPLICIT_DEF
More information about the llvm-commits
mailing list