[llvm] [AMDGPU][CoExecSched] Add free-proximity scheduling heuristic (PR #225334)
Romanov Vlad via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 07:15:07 PDT 2026
https://github.com/romanovvlad updated https://github.com/llvm/llvm-project/pull/225334
>From f8411d502b197c76f8bf832530dd5c42644fd003 Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Tue, 22 Sep 2026 02:52:13 -0500
Subject: [PATCH 1/2] [AMDGPU][CoExecSched] Add free-proximity scheduling
heuristic
Add a new scheduling heuristic that sorts PrioritySUs preferring
candidates closer to killing registers, reducing live register pressure.
Two-level comparison:
1. Prefer cand with more registers whose producers all have exactly 1
unscheduled successor (scheduling can free those registers immediately).
2. Prefer cand with lower min unscheduled successors among remaining
registers (closer to killing in future).
Unscheduled successors are computed manually as NumSuccsLeft is not
decremented in top-down scheduling.
When active, PrioritySUs are rebuilt every pick cycle to reflect the
current scheduling state.
By default (auto mode) the heuristic is enabled when VGPR pressure
approaches the excess limit, which is recomputed per each pick.
Option -amdgpu-coexec-reg-free-proximity={off,auto,always} controls the
behavior.
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 211 +++++++++++++++---
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 32 ++-
.../AMDGPU/coexec-reg-free-proximity.mir | 56 +++++
3 files changed, 261 insertions(+), 38 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index b728bb24f6758b..1a18b5d9dee743 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -42,6 +42,19 @@ static cl::opt<CarriedLatency> BlockCarriedLatency(
// Default VGPR threshold percent for coexec scheduler.
static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
+enum class RegFreeProximityMode { Off, Auto, Always };
+
+static cl::opt<RegFreeProximityMode> CoexecRegFreeProximity(
+ "amdgpu-coexec-reg-free-proximity", cl::Hidden,
+ cl::init(RegFreeProximityMode::Auto),
+ cl::desc("Prioritize instructions which are expected to free a register "
+ "sooner (lower min NumSuccsLeft)."),
+ cl::values(clEnumValN(RegFreeProximityMode::Off, "off", "Disabled."),
+ clEnumValN(RegFreeProximityMode::Auto, "auto",
+ "Enabled when HighPressure is set."),
+ clEnumValN(RegFreeProximityMode::Always, "always",
+ "Always enabled.")));
+
namespace {
// Used to disable post-RA scheduling with function level granularity.
@@ -467,32 +480,163 @@ SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
return TargetSU;
}
-void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
- if (!AllSUs.insert(SU))
- llvm_unreachable("HardwareUnit already contains SU!");
+int HardwareUnitInfo::compareDepth(SUnit *Candidate, SUnit *Existing) const {
+ const unsigned CurDepth = Existing->getDepth();
+ const unsigned CandDepth = Candidate->getDepth();
- TotalCycles += BlockingCycles;
+ if (CandDepth > CurDepth)
+ return -1;
+ if (CandDepth == CurDepth)
+ return 0;
+ return 1;
+}
+
+int HardwareUnitInfo::compareRegFreeProximity(SUnit *Candidate,
+ SUnit *Existing) const {
+ const MachineInstr *CandMI = Candidate->getInstr();
+ const bool IsMemOp = SIInstrInfo::isDS(*CandMI) ||
+ SIInstrInfo::isFLAT(*CandMI) ||
+ SIInstrInfo::isVMEM(*CandMI);
+ if (IsMemOp)
+ return compareDepth(Candidate, Existing);
+ auto getRegStats = [](const SUnit *SU) {
+ SmallDenseMap<Register, unsigned, 8> RegMaxUnsched;
+
+ for (const SDep &Pred : SU->Preds) {
+ if (Pred.getKind() != SDep::Data)
+ continue;
+ Register Reg = Pred.getReg();
+ if (!Reg)
+ continue;
+ const SUnit *PredSU = Pred.getSUnit();
+ unsigned Unscheduled = 0;
+ for (const SDep &Succ : PredSU->Succs)
+ if (Succ.getKind() == SDep::Data && !Succ.getSUnit()->isScheduled)
+ ++Unscheduled;
+
+ LLVM_DEBUG({
+ dbgs() << " pred SU(" << PredSU->NodeNum
+ << ") reg=" << printReg(Reg) << " unschedSuccs=" << Unscheduled
+ << " ";
+ if (PredSU->getInstr())
+ PredSU->getInstr()->print(dbgs(), /*IsStandalone=*/true,
+ /*SkipOpers=*/false, /*SkipDebugLoc=*/true);
+ else
+ dbgs() << "<no instr>";
+ dbgs() << "\n";
+ });
+ RegMaxUnsched[Reg] = std::max(RegMaxUnsched[Reg], Unscheduled);
+ }
+
+ unsigned Frees = 0;
+ unsigned MinOther =
+ RegMaxUnsched.empty() ? 0 : RegMaxUnsched.begin()->second;
+ for (auto &[Reg, MaxUnsched] : RegMaxUnsched) {
+ if (MaxUnsched < 2)
+ ++Frees;
+ else
+ MinOther = std::min(MinOther, MaxUnsched);
+ }
+ LLVM_DEBUG(dbgs() << " => frees=" << Frees
+ << " minOther=" << MinOther
+ << " (regs=" << RegMaxUnsched.size() << ")\n");
+ return std::pair(Frees, MinOther);
+ };
+
+ LLVM_DEBUG(dbgs() << " RegFreeProximity: Existing SU("
+ << Existing->NodeNum << ") vs Candidate SU("
+ << Candidate->NodeNum << ")\n");
+ auto [CurFrees, CurMinOther] = getRegStats(Existing);
+ auto [CandFrees, CandMinOther] = getRegStats(Candidate);
+
+ LLVM_DEBUG(dbgs() << " Existing: frees=" << CurFrees << " minOther="
+ << CurMinOther << " Candidate: frees=" << CandFrees
+ << " minOther=" << CandMinOther << "\n");
+
+ if (CandFrees > CurFrees) {
+ LLVM_DEBUG(dbgs() << " -> Candidate wins (more frees)\n");
+ return 1;
+ }
+ if (CandFrees < CurFrees) {
+ LLVM_DEBUG(dbgs() << " -> Existing wins (more frees)\n");
+ return -1;
+ }
+
+ if (CandMinOther < CurMinOther) {
+ LLVM_DEBUG(dbgs() << " -> Candidate wins (closer to freeing)\n");
+ return 1;
+ }
+ if (CandMinOther > CurMinOther) {
+ LLVM_DEBUG(dbgs() << " -> Existing wins (closer to freeing)\n");
+ return -1;
+ }
+ LLVM_DEBUG(dbgs() << " -> Tie\n");
+ return 0;
+}
+
+void HardwareUnitInfo::updatePrioritySUsWith(SUnit *Cand,
+ bool IsCloseToRegPressureLimit) {
if (PrioritySUs.empty()) {
- PrioritySUs.insert(SU);
+ PrioritySUs.insert(Cand);
return;
}
- unsigned SUDepth = SU->getDepth();
- unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
- if (SUDepth > CurrDepth)
+
+ int Decision = 0;
+
+ SUnit *Existing = *PrioritySUs.begin();
+ bool UseRegFree = CoexecRegFreeProximity == RegFreeProximityMode::Always ||
+ (CoexecRegFreeProximity == RegFreeProximityMode::Auto &&
+ IsCloseToRegPressureLimit);
+
+ LLVM_DEBUG(dbgs() << " updatePrioritySUs: SU(" << Cand->NodeNum
+ << ") vs existing SU(" << Existing->NodeNum << ")"
+ << " mode=" << (UseRegFree ? "RegFreeProximity" : "Depth")
+ << "\n");
+
+ if (!UseRegFree)
+ Decision = compareDepth(Cand, Existing);
+ else
+ Decision = compareRegFreeProximity(Cand, Existing);
+
+ LLVM_DEBUG(dbgs() << " decision=" << Decision
+ << (Decision > 0 ? " (candidate better)"
+ : Decision < 0 ? " (existing better)"
+ : " (tie)")
+ << "\n");
+
+ if (Decision < 0)
return;
- if (SUDepth == CurrDepth) {
- PrioritySUs.insert(SU);
+ if (Decision == 0) {
+ PrioritySUs.insert(Cand);
return;
}
- // SU is lower depth and should be prioritized.
PrioritySUs.clear();
- PrioritySUs.insert(SU);
+ PrioritySUs.insert(Cand);
+}
+
+void HardwareUnitInfo::rebuildPrioritySUs(bool IsCloseToRegPressureLimit) {
+ if (AllSUs.empty())
+ return;
+ PrioritySUs.clear();
+ for (auto *SU : AllSUs)
+ updatePrioritySUsWith(SU, IsCloseToRegPressureLimit);
}
-void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
+void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles,
+ bool IsCloseToRegPressureLimit) {
+ if (!AllSUs.insert(SU))
+ llvm_unreachable("HardwareUnit already contains SU!");
+
+ TotalCycles += BlockingCycles;
+
+ updatePrioritySUsWith(SU, IsCloseToRegPressureLimit);
+}
+
+void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles,
+ bool IsCloseToRegPressureLimit) {
// We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
// we just clear the HWUI. However, we still have instructions which map to
// this HWUI. Don't bother managing the state for these HWUI.
@@ -510,27 +654,8 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
if (AllSUs.empty())
return;
- if (PrioritySUs.empty()) {
- for (auto SU : AllSUs) {
- if (PrioritySUs.empty()) {
- PrioritySUs.insert(SU);
- continue;
- }
- unsigned SUDepth = SU->getDepth();
- unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
- if (SUDepth > CurrDepth)
- continue;
-
- if (SUDepth == CurrDepth) {
- PrioritySUs.insert(SU);
- continue;
- }
-
- // SU is lower depth and should be prioritized.
- PrioritySUs.clear();
- PrioritySUs.insert(SU);
- }
- }
+ if (PrioritySUs.empty())
+ rebuildPrioritySUs(IsCloseToRegPressureLimit);
}
void HardwareUnitInfo::finalizeCycles() {
@@ -603,7 +728,7 @@ void CandidateHeuristics::updateForScheduling(SUnit *SU) {
HardwareUnitInfo *HWUI =
getHWUIFromFlavor(classifyFlavor(*SU->getInstr(), *SII));
assert(HWUI);
- HWUI->markScheduled(SU, getHWUICyclesForSU(SU));
+ HWUI->markScheduled(SU, getHWUICyclesForSU(SU), IsCloseToRegPressureLimit);
}
void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
@@ -722,7 +847,8 @@ void CandidateHeuristics::collectRegionSummary() {
for (auto &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
const InstructionFlavor Flavor = classifyFlavor(*MI, *SII);
- HWUInfo[static_cast<int>(Flavor)].insert(&SU, getHWUICyclesForSU(&SU));
+ HWUInfo[static_cast<int>(Flavor)].insert(&SU, getHWUICyclesForSU(&SU),
+ IsCloseToRegPressureLimit);
unsigned CarriedLatency = getCarriedLatency(&SU);
if (CarriedLatency)
CarriedLatencies[MI] = CarriedLatency;
@@ -1198,6 +1324,19 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
}
}
+ constexpr unsigned MaxVGPRPressureInc = 16;
+ constexpr unsigned MaxVGPRPressureIncFactor = 2;
+ const bool IsCloseToRegPressureLimit =
+ DAG->isTrackingPressure() &&
+ VGPRPressure + MaxVGPRPressureIncFactor * MaxVGPRPressureInc >=
+ VGPRExcessLimit;
+ LLVM_DEBUG(dbgs() << "IsCloseToRegPressureLimit=" << IsCloseToRegPressureLimit
+ << " (VGPR=" << VGPRPressure << " limit=" << VGPRExcessLimit
+ << ")\n");
+ Heurs.setIsCloseToRegPressureLimit(IsCloseToRegPressureLimit);
+ if (IsCloseToRegPressureLimit)
+ Heurs.rebuildAllPrioritySUs();
+
auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
for (SUnit *SU : Q) {
SchedCandidate TryCand(ZonePolicy);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 6ce5aaeda8c82e..38dbe9a3ee1154 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -107,7 +107,20 @@ class HardwareUnitInfo {
/// behavior which is not modelled in the compiler.
unsigned BufferCycles = 0;
+ /// Compares two SUnits by depth (lower depth = higher priority for top-down).
+ /// \returns -1 if Candidate is worse, 0 if equal, 1 if Candidate is better.
+ int compareDepth(SUnit *Candidate, SUnit *Existing) const;
+
+ /// Compares two SUnits by proximity to freeing a register.
+ /// \returns -1 if Candidate is worse, 0 if equal, 1 if Candidate is better.
+ int compareRegFreeProximity(SUnit *Candidate, SUnit *Existing) const;
+
+ /// Try to update PrioritySUs with a new \p SU.
+ void updatePrioritySUsWith(SUnit *SU, bool IsCloseToRegPressureLimit = false);
+
public:
+ /// Rebuild PrioritySUs from AllSUs using the given pressure flag.
+ void rebuildPrioritySUs(bool IsCloseToRegPressureLimit);
HardwareUnitInfo() {}
unsigned size() { return AllSUs.size(); }
@@ -192,11 +205,13 @@ class HardwareUnitInfo {
SUnit *getNextTargetSU(bool LookDeep = false) const;
/// Insert the \p SU into AllSUs and account its \p BlockingCycles into
/// the TotalCycles. This maintains the list of PrioritySUs.
- void insert(SUnit *SU, unsigned BlockingCycles);
+ void insert(SUnit *SU, unsigned BlockingCycles,
+ bool IsCloseToRegPressureLimit);
/// Update the state for \p SU being scheduled by removing it from the AllSUs
/// and reducing its \p BlockingCycles from the TotalCycles. This maintains
/// the list of PrioritySUs.
- void markScheduled(SUnit *SU, unsigned BlockingCycles);
+ void markScheduled(SUnit *SU, unsigned BlockingCycles,
+ bool IsCloseToRegPressureLimit);
/// After we've collected all the region pressure for this HWUI, correct for
/// any specifics of the behavior of this resource. For example, if the
/// HardwareUnit can hold N instructions simultaneously, then there is no
@@ -255,6 +270,10 @@ class CandidateHeuristics {
StallCosts getStallCosts(SUnit *SU, SchedBoundary &Zone);
+ /// Controls whether or not the KillProximity heuristic is used when
+ /// selecting the next candidate SU for scheduling.
+ bool IsCloseToRegPressureLimit = false;
+
public:
CandidateHeuristics() = default;
@@ -310,6 +329,15 @@ class CandidateHeuristics {
SchedBoundary *Zone) const;
void dumpRegionSummary();
+
+ void setIsCloseToRegPressureLimit(bool Value) {
+ IsCloseToRegPressureLimit = Value;
+ }
+
+ void rebuildAllPrioritySUs() {
+ for (auto &HWUI : HWUInfo)
+ HWUI.rebuildPrioritySUs(IsCloseToRegPressureLimit);
+ }
};
class AMDGPUCoExecSchedStrategy final : public GCNSchedStrategy {
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
new file mode 100644
index 00000000000000..37f89fbb92f313
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
@@ -0,0 +1,56 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -stress-regalloc=8 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck %s
+
+# Test the register free proximity heuristic in the coexec scheduler.
+#
+# %s0 and %s1 are used by add_a1, add_a2, add_a3 (3 users each).
+# %s2 and %s3 are used by add_b1, add_b2, add_b3 (3 users each).
+# All six adds are independent and ready simultaneously.
+#
+# Without the heuiristic: scheduled in program order (a1, b1, a2, b2, a3, b3).
+# With the heuristic: it groups adds by operand so that registers are freed
+# sooner.
+
+--- |
+ define void @free_proximity() #0 { ret void }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+
+name: free_proximity
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+ ; CHECK-LABEL: name: free_proximity
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %s0:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: %s1:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: %s2:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: %s3:vgpr_32 = COPY $vgpr3
+ ; CHECK-NEXT: %add_a1:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ ; CHECK-NEXT: %add_a2:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ ; CHECK-NEXT: %add_a3:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ ; CHECK-NEXT: %add_b1:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ ; CHECK-NEXT: %add_b2:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ ; CHECK-NEXT: %add_b3:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %add_a1, implicit %add_b1, implicit %add_a2, implicit %add_b2, implicit %add_a3, implicit %add_b3
+ %s0:vgpr_32 = COPY $vgpr0
+ %s1:vgpr_32 = COPY $vgpr1
+ %s2:vgpr_32 = COPY $vgpr2
+ %s3:vgpr_32 = COPY $vgpr3
+
+ %add_a1:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ %add_b1:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ %add_a2:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ %add_b2:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ %add_a3:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
+ %add_b3:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+
+ S_ENDPGM 0, implicit %add_a1, implicit %add_b1, implicit %add_a2, implicit %add_b2, implicit %add_a3, implicit %add_b3
+...
>From 929361b7ed7e889b5ff4a2f4844f8bc304a32022 Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Thu, 24 Sep 2026 08:46:11 -0500
Subject: [PATCH 2/2] Enable by default
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 221 ++++--------
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 39 +--
.../AMDGPU/coexec-mfma-interleave-gfx950.ll | 12 +-
.../coexec-reg-free-proximity-multi-def.mir | 56 ++++
.../AMDGPU/coexec-reg-free-proximity.mir | 55 +--
.../AMDGPU/coexec-sched-agpr-excess.mir | 2 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 316 +++++++++---------
.../llvm.amdgcn.sched.group.barrier.gfx12.ll | 2 +-
8 files changed, 335 insertions(+), 368 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity-multi-def.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 1a18b5d9dee743..0ae6cabbae9222 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -42,19 +42,6 @@ static cl::opt<CarriedLatency> BlockCarriedLatency(
// Default VGPR threshold percent for coexec scheduler.
static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
-enum class RegFreeProximityMode { Off, Auto, Always };
-
-static cl::opt<RegFreeProximityMode> CoexecRegFreeProximity(
- "amdgpu-coexec-reg-free-proximity", cl::Hidden,
- cl::init(RegFreeProximityMode::Auto),
- cl::desc("Prioritize instructions which are expected to free a register "
- "sooner (lower min NumSuccsLeft)."),
- cl::values(clEnumValN(RegFreeProximityMode::Off, "off", "Disabled."),
- clEnumValN(RegFreeProximityMode::Auto, "auto",
- "Enabled when HighPressure is set."),
- clEnumValN(RegFreeProximityMode::Always, "always",
- "Always enabled.")));
-
namespace {
// Used to disable post-RA scheduling with function level granularity.
@@ -480,163 +467,109 @@ SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
return TargetSU;
}
-int HardwareUnitInfo::compareDepth(SUnit *Candidate, SUnit *Existing) const {
- const unsigned CurDepth = Existing->getDepth();
- const unsigned CandDepth = Candidate->getDepth();
-
- if (CandDepth > CurDepth)
- return -1;
- if (CandDepth == CurDepth)
- return 0;
- return 1;
-}
-
-int HardwareUnitInfo::compareRegFreeProximity(SUnit *Candidate,
- SUnit *Existing) const {
- const MachineInstr *CandMI = Candidate->getInstr();
- const bool IsMemOp = SIInstrInfo::isDS(*CandMI) ||
- SIInstrInfo::isFLAT(*CandMI) ||
- SIInstrInfo::isVMEM(*CandMI);
- if (IsMemOp)
- return compareDepth(Candidate, Existing);
-
- auto getRegStats = [](const SUnit *SU) {
- SmallDenseMap<Register, unsigned, 8> RegMaxUnsched;
-
- for (const SDep &Pred : SU->Preds) {
- if (Pred.getKind() != SDep::Data)
- continue;
- Register Reg = Pred.getReg();
- if (!Reg)
- continue;
- const SUnit *PredSU = Pred.getSUnit();
- unsigned Unscheduled = 0;
- for (const SDep &Succ : PredSU->Succs)
- if (Succ.getKind() == SDep::Data && !Succ.getSUnit()->isScheduled)
- ++Unscheduled;
-
- LLVM_DEBUG({
- dbgs() << " pred SU(" << PredSU->NodeNum
- << ") reg=" << printReg(Reg) << " unschedSuccs=" << Unscheduled
- << " ";
- if (PredSU->getInstr())
- PredSU->getInstr()->print(dbgs(), /*IsStandalone=*/true,
- /*SkipOpers=*/false, /*SkipDebugLoc=*/true);
- else
- dbgs() << "<no instr>";
- dbgs() << "\n";
- });
- RegMaxUnsched[Reg] = std::max(RegMaxUnsched[Reg], Unscheduled);
- }
-
- unsigned Frees = 0;
- unsigned MinOther =
- RegMaxUnsched.empty() ? 0 : RegMaxUnsched.begin()->second;
- for (auto &[Reg, MaxUnsched] : RegMaxUnsched) {
- if (MaxUnsched < 2)
- ++Frees;
+/// Counts the registers scheduling \p SU frees, i.e. those it reads whose
+/// defining SU has \p SU as its only remaining unscheduled data successor.
+///
+/// TODO: weight by register size and split per bank (ArchVGPR/AGPR/SGPR have
+/// separate budgets), charging SU for its own defs, so the result is a signed
+/// pressure delta rather than a count.
+static unsigned countRegFrees(const SUnit *SU) {
+ SmallDenseMap<Register, unsigned, 8> RegMaxUnsched;
+
+ for (const SDep &Pred : SU->Preds) {
+ if (Pred.getKind() != SDep::Data)
+ continue;
+ Register Reg = Pred.getReg();
+ if (!Reg)
+ continue;
+ const SUnit *PredSU = Pred.getSUnit();
+ unsigned Unscheduled = 0;
+ // Only successors that read Reg keep it live; PredSU may define other
+ // registers whose edges say nothing about when Reg dies.
+ for (const SDep &Succ : PredSU->Succs)
+ if (Succ.getKind() == SDep::Data && Succ.getReg() == Reg &&
+ !Succ.getSUnit()->isScheduled)
+ ++Unscheduled;
+
+ LLVM_DEBUG({
+ dbgs() << " pred SU(" << PredSU->NodeNum
+ << ") reg=" << printReg(Reg) << " unschedSuccs=" << Unscheduled
+ << " ";
+ if (PredSU->getInstr())
+ PredSU->getInstr()->print(dbgs(), /*IsStandalone=*/true,
+ /*SkipOpers=*/false, /*SkipDebugLoc=*/true);
else
- MinOther = std::min(MinOther, MaxUnsched);
- }
- LLVM_DEBUG(dbgs() << " => frees=" << Frees
- << " minOther=" << MinOther
- << " (regs=" << RegMaxUnsched.size() << ")\n");
- return std::pair(Frees, MinOther);
- };
-
- LLVM_DEBUG(dbgs() << " RegFreeProximity: Existing SU("
- << Existing->NodeNum << ") vs Candidate SU("
- << Candidate->NodeNum << ")\n");
- auto [CurFrees, CurMinOther] = getRegStats(Existing);
- auto [CandFrees, CandMinOther] = getRegStats(Candidate);
-
- LLVM_DEBUG(dbgs() << " Existing: frees=" << CurFrees << " minOther="
- << CurMinOther << " Candidate: frees=" << CandFrees
- << " minOther=" << CandMinOther << "\n");
-
- if (CandFrees > CurFrees) {
- LLVM_DEBUG(dbgs() << " -> Candidate wins (more frees)\n");
- return 1;
- }
- if (CandFrees < CurFrees) {
- LLVM_DEBUG(dbgs() << " -> Existing wins (more frees)\n");
- return -1;
+ dbgs() << "<no instr>";
+ dbgs() << "\n";
+ });
+ auto &MaxUnsched = RegMaxUnsched[Reg];
+ MaxUnsched = std::max(MaxUnsched, Unscheduled);
}
- if (CandMinOther < CurMinOther) {
- LLVM_DEBUG(dbgs() << " -> Candidate wins (closer to freeing)\n");
- return 1;
- }
- if (CandMinOther > CurMinOther) {
- LLVM_DEBUG(dbgs() << " -> Existing wins (closer to freeing)\n");
- return -1;
- }
- LLVM_DEBUG(dbgs() << " -> Tie\n");
- return 0;
+ // SU itself is always counted, so MaxUnsched >= 1 and freed means == 1.
+ unsigned Frees = 0;
+ for (const auto &[Reg, MaxUnsched] : RegMaxUnsched)
+ if (MaxUnsched < 2)
+ ++Frees;
+ LLVM_DEBUG(dbgs() << " => frees=" << Frees
+ << " (regs=" << RegMaxUnsched.size() << ")\n");
+ return Frees;
}
-void HardwareUnitInfo::updatePrioritySUsWith(SUnit *Cand,
- bool IsCloseToRegPressureLimit) {
+void HardwareUnitInfo::updatePrioritySUsWith(SUnit *Cand) {
if (PrioritySUs.empty()) {
PrioritySUs.insert(Cand);
return;
}
- int Decision = 0;
-
SUnit *Existing = *PrioritySUs.begin();
- bool UseRegFree = CoexecRegFreeProximity == RegFreeProximityMode::Always ||
- (CoexecRegFreeProximity == RegFreeProximityMode::Auto &&
- IsCloseToRegPressureLimit);
LLVM_DEBUG(dbgs() << " updatePrioritySUs: SU(" << Cand->NodeNum
- << ") vs existing SU(" << Existing->NodeNum << ")"
- << " mode=" << (UseRegFree ? "RegFreeProximity" : "Depth")
- << "\n");
+ << ") vs existing SU(" << Existing->NodeNum << ")\n");
- if (!UseRegFree)
- Decision = compareDepth(Cand, Existing);
- else
- Decision = compareRegFreeProximity(Cand, Existing);
-
- LLVM_DEBUG(dbgs() << " decision=" << Decision
- << (Decision > 0 ? " (candidate better)"
- : Decision < 0 ? " (existing better)"
- : " (tie)")
- << "\n");
+ if (PriorityHeadFreesFor != Existing) {
+ PriorityHeadFrees = countRegFrees(Existing);
+ PriorityHeadFreesFor = Existing;
+ }
+ unsigned CandFrees = countRegFrees(Cand);
+ LLVM_DEBUG(dbgs() << " Existing: frees=" << PriorityHeadFrees
+ << " Candidate: frees=" << CandFrees << "\n");
- if (Decision < 0)
+ if (CandFrees < PriorityHeadFrees)
return;
- if (Decision == 0) {
+ // Keep every tied SU so the candidate heuristics can pick among them.
+ if (CandFrees == PriorityHeadFrees) {
PrioritySUs.insert(Cand);
return;
}
PrioritySUs.clear();
PrioritySUs.insert(Cand);
+ // Cand is the new head; reuse the count just computed for it.
+ PriorityHeadFrees = CandFrees;
+ PriorityHeadFreesFor = Cand;
}
-void HardwareUnitInfo::rebuildPrioritySUs(bool IsCloseToRegPressureLimit) {
+void HardwareUnitInfo::rebuildPrioritySUs() {
if (AllSUs.empty())
return;
PrioritySUs.clear();
+ PriorityHeadFreesFor = nullptr;
for (auto *SU : AllSUs)
- updatePrioritySUsWith(SU, IsCloseToRegPressureLimit);
+ updatePrioritySUsWith(SU);
}
-void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles,
- bool IsCloseToRegPressureLimit) {
+void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
if (!AllSUs.insert(SU))
llvm_unreachable("HardwareUnit already contains SU!");
TotalCycles += BlockingCycles;
- updatePrioritySUsWith(SU, IsCloseToRegPressureLimit);
+ updatePrioritySUsWith(SU);
}
-void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles,
- bool IsCloseToRegPressureLimit) {
+void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
// We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
// we just clear the HWUI. However, we still have instructions which map to
// this HWUI. Don't bother managing the state for these HWUI.
@@ -646,6 +579,8 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles,
ScheduledSUs.push_back(SU);
AllSUs.remove(SU);
PrioritySUs.remove(SU);
+ // The cached count was derived from unscheduled-successor counts SU changes.
+ PriorityHeadFreesFor = nullptr;
// BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
// either case, each SU uses the HardwareUnit for BlockingCycles.
@@ -655,7 +590,7 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles,
if (AllSUs.empty())
return;
if (PrioritySUs.empty())
- rebuildPrioritySUs(IsCloseToRegPressureLimit);
+ rebuildPrioritySUs();
}
void HardwareUnitInfo::finalizeCycles() {
@@ -728,7 +663,7 @@ void CandidateHeuristics::updateForScheduling(SUnit *SU) {
HardwareUnitInfo *HWUI =
getHWUIFromFlavor(classifyFlavor(*SU->getInstr(), *SII));
assert(HWUI);
- HWUI->markScheduled(SU, getHWUICyclesForSU(SU), IsCloseToRegPressureLimit);
+ HWUI->markScheduled(SU, getHWUICyclesForSU(SU));
}
void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
@@ -847,8 +782,7 @@ void CandidateHeuristics::collectRegionSummary() {
for (auto &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
const InstructionFlavor Flavor = classifyFlavor(*MI, *SII);
- HWUInfo[static_cast<int>(Flavor)].insert(&SU, getHWUICyclesForSU(&SU),
- IsCloseToRegPressureLimit);
+ HWUInfo[static_cast<int>(Flavor)].insert(&SU, getHWUICyclesForSU(&SU));
unsigned CarriedLatency = getCarriedLatency(&SU);
if (CarriedLatency)
CarriedLatencies[MI] = CarriedLatency;
@@ -1324,18 +1258,9 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
}
}
- constexpr unsigned MaxVGPRPressureInc = 16;
- constexpr unsigned MaxVGPRPressureIncFactor = 2;
- const bool IsCloseToRegPressureLimit =
- DAG->isTrackingPressure() &&
- VGPRPressure + MaxVGPRPressureIncFactor * MaxVGPRPressureInc >=
- VGPRExcessLimit;
- LLVM_DEBUG(dbgs() << "IsCloseToRegPressureLimit=" << IsCloseToRegPressureLimit
- << " (VGPR=" << VGPRPressure << " limit=" << VGPRExcessLimit
- << ")\n");
- Heurs.setIsCloseToRegPressureLimit(IsCloseToRegPressureLimit);
- if (IsCloseToRegPressureLimit)
- Heurs.rebuildAllPrioritySUs();
+ // countRegFrees() depends on what is already scheduled, so the ranking goes
+ // stale after every pick.
+ Heurs.rebuildAllPrioritySUs();
auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
for (SUnit *SU : Q) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 38dbe9a3ee1154..915bfe6b3902df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -70,8 +70,8 @@ class HardwareUnitInfo {
/// tryCriticalResourceDependency and tryCriticalResource: we schedule the
/// dependencies for a SU on critical resource, then schedule that same SU on
/// the critical resource. This agreement results in shorter live ranges and
- /// more regular HardwareUnit access patterns. SUs are prioritized based on
- /// depth for top-down scheduling.
+ /// more regular HardwareUnit access patterns. SUs are prioritized by how
+ /// many registers scheduling them frees; see countRegFrees().
SmallSetVector<SUnit *, 16> PrioritySUs;
/// All the SUs in the region that consume this resource.
SmallSetVector<SUnit *, 16> AllSUs;
@@ -107,22 +107,20 @@ class HardwareUnitInfo {
/// behavior which is not modelled in the compiler.
unsigned BufferCycles = 0;
- /// Compares two SUnits by depth (lower depth = higher priority for top-down).
- /// \returns -1 if Candidate is worse, 0 if equal, 1 if Candidate is better.
- int compareDepth(SUnit *Candidate, SUnit *Existing) const;
-
- /// Compares two SUnits by proximity to freeing a register.
- /// \returns -1 if Candidate is worse, 0 if equal, 1 if Candidate is better.
- int compareRegFreeProximity(SUnit *Candidate, SUnit *Existing) const;
+ /// Cached countRegFrees() of the head of PrioritySUs, valid only while
+ /// PriorityHeadFreesFor is still the head.
+ const SUnit *PriorityHeadFreesFor = nullptr;
+ unsigned PriorityHeadFrees = 0;
/// Try to update PrioritySUs with a new \p SU.
- void updatePrioritySUsWith(SUnit *SU, bool IsCloseToRegPressureLimit = false);
+ void updatePrioritySUsWith(SUnit *SU);
public:
- /// Rebuild PrioritySUs from AllSUs using the given pressure flag.
- void rebuildPrioritySUs(bool IsCloseToRegPressureLimit);
HardwareUnitInfo() {}
+ /// Rebuild PrioritySUs from AllSUs.
+ void rebuildPrioritySUs();
+
unsigned size() { return AllSUs.size(); }
unsigned getTotalCycles() { return TotalCycles; }
@@ -185,6 +183,7 @@ class HardwareUnitInfo {
void reset() {
AllSUs.clear();
PrioritySUs.clear();
+ PriorityHeadFreesFor = nullptr;
ScheduledSUs.clear();
TotalCycles = 0;
Type = AMDGPU::InstructionFlavor::Other;
@@ -205,13 +204,11 @@ class HardwareUnitInfo {
SUnit *getNextTargetSU(bool LookDeep = false) const;
/// Insert the \p SU into AllSUs and account its \p BlockingCycles into
/// the TotalCycles. This maintains the list of PrioritySUs.
- void insert(SUnit *SU, unsigned BlockingCycles,
- bool IsCloseToRegPressureLimit);
+ void insert(SUnit *SU, unsigned BlockingCycles);
/// Update the state for \p SU being scheduled by removing it from the AllSUs
/// and reducing its \p BlockingCycles from the TotalCycles. This maintains
/// the list of PrioritySUs.
- void markScheduled(SUnit *SU, unsigned BlockingCycles,
- bool IsCloseToRegPressureLimit);
+ void markScheduled(SUnit *SU, unsigned BlockingCycles);
/// After we've collected all the region pressure for this HWUI, correct for
/// any specifics of the behavior of this resource. For example, if the
/// HardwareUnit can hold N instructions simultaneously, then there is no
@@ -270,10 +267,6 @@ class CandidateHeuristics {
StallCosts getStallCosts(SUnit *SU, SchedBoundary &Zone);
- /// Controls whether or not the KillProximity heuristic is used when
- /// selecting the next candidate SU for scheduling.
- bool IsCloseToRegPressureLimit = false;
-
public:
CandidateHeuristics() = default;
@@ -330,13 +323,9 @@ class CandidateHeuristics {
void dumpRegionSummary();
- void setIsCloseToRegPressureLimit(bool Value) {
- IsCloseToRegPressureLimit = Value;
- }
-
void rebuildAllPrioritySUs() {
for (auto &HWUI : HWUInfo)
- HWUI.rebuildPrioritySUs(IsCloseToRegPressureLimit);
+ HWUI.rebuildPrioritySUs();
}
};
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-mfma-interleave-gfx950.ll b/llvm/test/CodeGen/AMDGPU/coexec-mfma-interleave-gfx950.ll
index 3ebfd4d1d5a24c..f95d5fbab422e0 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-mfma-interleave-gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-mfma-interleave-gfx950.ll
@@ -11,17 +11,17 @@ define amdgpu_kernel void @mfma_16x16_interleave(
; CHECK-LABEL: mfma_16x16_interleave:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CHECK-NEXT: v_and_b32_e32 v14, 0x3ff, v0
+; CHECK-NEXT: v_and_b32_e32 v16, 0x3ff, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 4, v16
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 4, v14
-; CHECK-NEXT: v_mul_hi_i32_i24_e32 v15, -12, v14
-; CHECK-NEXT: v_mul_i32_i24_e32 v14, -12, v14
+; CHECK-NEXT: v_mul_hi_i32_i24_e32 v17, -12, v16
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_lshl_add_u64 v[16:17], s[0:1], 0, v[0:1]
+; CHECK-NEXT: v_lshl_add_u64 v[14:15], s[0:1], 0, v[0:1]
+; CHECK-NEXT: v_mul_i32_i24_e32 v16, -12, v16
; CHECK-NEXT: global_load_dwordx4 v[2:5], v0, s[0:1]
; CHECK-NEXT: global_load_dwordx4 v[6:9], v0, s[0:1] offset:16
; CHECK-NEXT: global_load_dwordx4 v[10:13], v0, s[0:1] offset:32
-; CHECK-NEXT: v_lshl_add_u64 v[18:19], v[16:17], 0, v[14:15]
+; CHECK-NEXT: v_lshl_add_u64 v[18:19], v[14:15], 0, v[16:17]
; CHECK-NEXT: global_load_dwordx2 v[30:31], v[18:19], off offset:16
; CHECK-NEXT: global_load_dwordx4 v[14:17], v[18:19], off
; CHECK-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x8
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity-multi-def.mir b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity-multi-def.mir
new file mode 100644
index 00000000000000..822f8a55d311fb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity-multi-def.mir
@@ -0,0 +1,56 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck %s
+
+# A def's freed-register count must only look at successors reading that def.
+# V_ADD_CO_U32_e64 defines %sum (read by %a) and %carry (read by %k0/%k1/%k2);
+# counting the %carry readers against %sum would hide that %a frees it, and %a
+# would lose to %b, which frees %q0 but leaves %q1 live for %d.
+
+--- |
+ define void @multi_def() #0 { ret void }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+---
+name: multi_def
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+ ; CHECK-LABEL: name: multi_def
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: %y:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: %z:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: %q0:vgpr_32 = COPY $vgpr3
+ ; CHECK-NEXT: %q1:vgpr_32 = COPY $vgpr4
+ ; CHECK-NEXT: %sum:vgpr_32, %carry:sreg_64_xexec = V_ADD_CO_U32_e64 %x, %y, 0, implicit $exec
+ ; CHECK-NEXT: %a:vgpr_32 = V_ADD_U32_e32 %sum, %z, implicit $exec
+ ; CHECK-NEXT: %b:vgpr_32 = V_ADD_U32_e32 %q0, %q1, implicit $exec
+ ; CHECK-NEXT: %d:vgpr_32 = V_ADD_U32_e32 %q1, %q1, implicit $exec
+ ; CHECK-NEXT: %k0:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %carry, implicit $exec
+ ; CHECK-NEXT: %k1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 2, %carry, implicit $exec
+ ; CHECK-NEXT: %k2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3, %carry, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %a, implicit %b, implicit %d, implicit %k0, implicit %k1, implicit %k2
+ %x:vgpr_32 = COPY $vgpr0
+ %y:vgpr_32 = COPY $vgpr1
+ %z:vgpr_32 = COPY $vgpr2
+ %q0:vgpr_32 = COPY $vgpr3
+ %q1:vgpr_32 = COPY $vgpr4
+
+ ; %sum is read only by %a; %carry is read by three other instructions.
+ %sum:vgpr_32, %carry:sreg_64_xexec = V_ADD_CO_U32_e64 %x, %y, 0, implicit $exec
+
+ %a:vgpr_32 = V_ADD_U32_e32 %sum, %z, implicit $exec
+ %b:vgpr_32 = V_ADD_U32_e32 %q0, %q1, implicit $exec
+ %d:vgpr_32 = V_ADD_U32_e32 %q1, %q1, implicit $exec
+
+ %k0:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %carry, implicit $exec
+ %k1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 2, %carry, implicit $exec
+ %k2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3, %carry, implicit $exec
+
+ S_ENDPGM 0, implicit %a, implicit %b, implicit %d, implicit %k0, implicit %k1, implicit %k2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
index 37f89fbb92f313..605f76c2786ca4 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-reg-free-proximity.mir
@@ -1,15 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -stress-regalloc=8 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck %s
-# Test the register free proximity heuristic in the coexec scheduler.
-#
-# %s0 and %s1 are used by add_a1, add_a2, add_a3 (3 users each).
-# %s2 and %s3 are used by add_b1, add_b2, add_b3 (3 users each).
-# All six adds are independent and ready simultaneously.
-#
-# Without the heuiristic: scheduled in program order (a1, b1, a2, b2, a3, b3).
-# With the heuristic: it groups adds by operand so that registers are freed
-# sooner.
+# Six independent FMAs, all ready at once. a1/a2/a3 read %s0, %s1 and a private
+# %p1/%p2/%p3; b1/b2/b3 read %s2, %s3, %s4. Each a frees its %p, so the a group
+# outranks the b group and goes first, killing %s0 and %s1 at a3. Peak drops
+# from 11 VGPRs in program order to 9, the best of all 720 orderings.
--- |
define void @free_proximity() #0 { ret void }
@@ -24,33 +19,41 @@ machineFunctionInfo:
isEntryFunction: true
body: |
bb.0:
- liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
; CHECK-LABEL: name: free_proximity
- ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %s0:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: %s1:vgpr_32 = COPY $vgpr1
; CHECK-NEXT: %s2:vgpr_32 = COPY $vgpr2
; CHECK-NEXT: %s3:vgpr_32 = COPY $vgpr3
- ; CHECK-NEXT: %add_a1:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- ; CHECK-NEXT: %add_a2:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- ; CHECK-NEXT: %add_a3:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- ; CHECK-NEXT: %add_b1:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
- ; CHECK-NEXT: %add_b2:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
- ; CHECK-NEXT: %add_b3:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
- ; CHECK-NEXT: S_ENDPGM 0, implicit %add_a1, implicit %add_b1, implicit %add_a2, implicit %add_b2, implicit %add_a3, implicit %add_b3
+ ; CHECK-NEXT: %s4:vgpr_32 = COPY $vgpr4
+ ; CHECK-NEXT: %p1:vgpr_32 = COPY $vgpr5
+ ; CHECK-NEXT: %p2:vgpr_32 = COPY $vgpr6
+ ; CHECK-NEXT: %p3:vgpr_32 = COPY $vgpr7
+ ; CHECK-NEXT: %a1:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %a2:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %a3:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p3, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b1:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b2:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %b3:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %a1, implicit %b1, implicit %a2, implicit %b2, implicit %a3, implicit %b3
%s0:vgpr_32 = COPY $vgpr0
%s1:vgpr_32 = COPY $vgpr1
%s2:vgpr_32 = COPY $vgpr2
%s3:vgpr_32 = COPY $vgpr3
+ %s4:vgpr_32 = COPY $vgpr4
+ %p1:vgpr_32 = COPY $vgpr5
+ %p2:vgpr_32 = COPY $vgpr6
+ %p3:vgpr_32 = COPY $vgpr7
- %add_a1:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- %add_b1:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
- %add_a2:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- %add_b2:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
- %add_a3:vgpr_32 = V_ADD_U32_e32 %s0, %s1, implicit $exec
- %add_b3:vgpr_32 = V_ADD_U32_e32 %s2, %s3, implicit $exec
+ %a1:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p1, 0, 0, implicit $mode, implicit $exec
+ %b1:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
+ %a2:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p2, 0, 0, implicit $mode, implicit $exec
+ %b2:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
+ %a3:vgpr_32 = V_FMA_F32_e64 0, %s0, 0, %s1, 0, %p3, 0, 0, implicit $mode, implicit $exec
+ %b3:vgpr_32 = V_FMA_F32_e64 0, %s2, 0, %s3, 0, %s4, 0, 0, implicit $mode, implicit $exec
- S_ENDPGM 0, implicit %add_a1, implicit %add_b1, implicit %add_a2, implicit %add_b2, implicit %add_a3, implicit %add_b3
+ S_ENDPGM 0, implicit %a1, implicit %b1, implicit %a2, implicit %b2, implicit %a3, implicit %b3
...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
index 7e65714c45d18e..aa0d494c2ff21a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
@@ -42,10 +42,10 @@ body: |
; NO_LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
- ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3)
; NO_LIMIT-NEXT: S_ENDPGM 0
%s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3)
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 97a0d6880c6b6c..ad261a7ccf561e 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -41,43 +41,42 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v88, s2
+; COEXEC-NEXT: v_mov_b32_e32 v92, s2
; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
; COEXEC-NEXT: s_and_b32 s3, s0, exec_lo
; COEXEC-NEXT: s_cselect_b32 s3, 1, 0
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v88 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v88 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v88
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v88 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v88 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v88 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v88 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v88 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v88 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v88 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v88 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v88 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v88 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v88 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v88 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v88 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v92 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v92
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v92 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v92 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v92 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v92 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v92 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v92 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v92 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v92 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v92 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v92 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:960
; COEXEC-NEXT: s_cmp_lg_u32 s3, 1
; COEXEC-NEXT: s_wait_dscnt 0xc
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
; COEXEC-NEXT: s_wait_dscnt 0x8
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; COEXEC-NEXT: s_cbranch_scc1 .LBB0_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
@@ -294,65 +293,63 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v124, s7
+; COEXEC-NEXT: v_mov_b32_e32 v100, s7
; COEXEC-NEXT: s_cmp_lg_u32 s9, 1
-; COEXEC-NEXT: v_mov_b32_e32 v156, s8
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v124
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v124 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v156
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v156 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v124 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v124 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v156 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v156 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v124 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v124 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v156 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v156 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v124 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v124 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v156 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v156 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384
+; COEXEC-NEXT: v_mov_b32_e32 v140, s8
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v100
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v100 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v140
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v140 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v100 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v100 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v100 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v100 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v100 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v100 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v100 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v100 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v100 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v100 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v100 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v100 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v100 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v100 offset:960
+; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v140 offset:128
; COEXEC-NEXT: s_wait_dscnt 0xf
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
-; COEXEC-NEXT: s_wait_dscnt 0x12
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
-; COEXEC-NEXT: s_wait_dscnt 0x14
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; COEXEC-NEXT: s_wait_dscnt 0x10
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v140 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v140 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v140 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v140 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v140 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v140 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v140 offset:192
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v140 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v140 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v140 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v140 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v140 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v140 offset:960
+; COEXEC-NEXT: s_wait_dscnt 0xb
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[112:119], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[112:119], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x6
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[48:55], v[104:111], v[24:31]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[48:55], v[104:111], v[24:31]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[64:71], v[32:39], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[64:71], v[32:39], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[120:127], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[120:127], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x2
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[80:87], v[40:47], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[80:87], v[40:47], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[128:135], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[128:135], v[0:7]
; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[96:103], v[136:143], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[96:103], v[136:143], v[0:7]
; COEXEC-NEXT: s_cbranch_scc1 .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
@@ -613,7 +610,7 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; COEXEC-NEXT: s_mov_b32 s8, 0
-; COEXEC-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
+; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
; COEXEC-NEXT: s_mov_b32 s9, s8
; COEXEC-NEXT: s_mov_b32 s10, s8
; COEXEC-NEXT: s_mov_b32 s11, s8
@@ -626,34 +623,34 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
+; COEXEC-NEXT: v_dual_mov_b32 v22, s2 :: v_dual_mov_b32 v7, v0
+; COEXEC-NEXT: v_dual_mov_b32 v8, v0 :: v_dual_mov_b32 v9, v0
+; COEXEC-NEXT: v_dual_mov_b32 v10, v0 :: v_dual_mov_b32 v11, v0
+; COEXEC-NEXT: v_dual_mov_b32 v12, v0 :: v_dual_mov_b32 v13, v0
+; COEXEC-NEXT: v_dual_mov_b32 v14, v0 :: v_dual_mov_b32 v15, v0
+; COEXEC-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v0
+; COEXEC-NEXT: v_dual_mov_b32 v18, v0 :: v_dual_mov_b32 v19, v0
+; COEXEC-NEXT: v_dual_mov_b32 v20, v0 :: v_dual_mov_b32 v21, v0
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v22
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v22 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v22 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v22 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v22 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v22 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v22 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v22 offset:448
+; COEXEC-NEXT: v_dual_mov_b32 v22, v0 :: v_dual_mov_b32 v23, v0
+; COEXEC-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v25, v0
+; COEXEC-NEXT: v_dual_mov_b32 v26, v0 :: v_dual_mov_b32 v27, v0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
+; COEXEC-NEXT: v_mov_b32_e32 v28, v0
+; COEXEC-NEXT: s_cselect_b32 s3, -1, 0
; COEXEC-NEXT: v_mov_b32_e32 v29, v0
-; COEXEC-NEXT: s_cselect_b32 s2, -1, 0
+; COEXEC-NEXT: s_add_co_i32 s0, s2, s1
; COEXEC-NEXT: v_mov_b32_e32 v30, v0
-; COEXEC-NEXT: s_add_co_i32 s0, s3, s1
+; COEXEC-NEXT: s_xor_b32 s2, s3, -1
; COEXEC-NEXT: v_mov_b32_e32 v31, v0
-; COEXEC-NEXT: s_xor_b32 s2, s2, -1
-; COEXEC-NEXT: v_mov_b32_e32 v60, s3
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v60
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v60 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v60 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v60 offset:448
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: .LBB2_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -671,27 +668,26 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; COEXEC-NEXT: s_and_b32 s3, s2, exec_lo
; COEXEC-NEXT: s_cselect_b32 s3, 1, 0
; COEXEC-NEXT: s_cmp_lg_u32 s3, 1
-; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[64:71], v[64:71], v[0:7]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v72
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v72 offset:64
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v72 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v72 offset:192
+; COEXEC-NEXT: s_wait_dscnt 0x4
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v72 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v72 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v72 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v72 offset:448
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[64:71], v[64:71], v[0:7]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[64:71], v[64:71], v[0:7]
; COEXEC-NEXT: s_cbranch_scc1 .LBB2_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_dscnt 0x7
-; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
@@ -935,41 +931,41 @@ define amdgpu_kernel void @ds_wmma_loop_carried(ptr addrspace(3) %base, ptr addr
; COEXEC-NEXT: s_add_co_i32 s0, s2, s1
; COEXEC-NEXT: ds_load_tr16_b128 v[0:3], v40
; COEXEC-NEXT: ds_load_tr16_b128 v[4:7], v40 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v40 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v40 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v40 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v40 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v40 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v40 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v40 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v40 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v40 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v40 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v40 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v40 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v40 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v40 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v40 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v40 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v40 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v40 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v40 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v40 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v40 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v40 offset:832
; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v40 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v40 offset:960
; COEXEC-NEXT: s_xor_b32 s2, s3, -1
; COEXEC-NEXT: s_wait_dscnt 0xe
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[0:7], v[64:71], 0
; COEXEC-NEXT: s_wait_dscnt 0xc
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[8:15], v[64:71], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[8:15], v[64:71], v[56:63]
; COEXEC-NEXT: s_wait_dscnt 0xa
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[32:39], v[64:71], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[16:23], v[64:71], 0
; COEXEC-NEXT: s_wait_dscnt 0x8
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[72:79], v[64:71], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[24:31], v[64:71], v[48:55]
; COEXEC-NEXT: s_wait_dscnt 0x6
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[16:23], v[64:71], v[56:63]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[32:39], v[64:71], 0
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[24:31], v[64:71], v[48:55]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[72:79], v[64:71], v[40:47]
; COEXEC-NEXT: s_wait_dscnt 0x2
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[80:87], v[64:71], v[40:47]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[80:87], v[64:71], 0
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[88:95], v[64:71], v[32:39]
; COEXEC-NEXT: .LBB3_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[0:7], v[16:23], v[56:63]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[0:7], v[8:15], v[56:63]
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
@@ -982,27 +978,26 @@ define amdgpu_kernel void @ds_wmma_loop_carried(ptr addrspace(3) %base, ptr addr
; COEXEC-NEXT: s_and_b32 s3, s2, exec_lo
; COEXEC-NEXT: s_cselect_b32 s3, 1, 0
; COEXEC-NEXT: s_cmp_lg_u32 s3, 1
-; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[8:15], v[24:31], v[48:55]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[64:71], v[64:71], v[40:47]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[64:71], v[64:71], v[32:39]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[0:7], v[16:23], v[56:63]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[56:63], v[0:7], v[8:15], v[56:63]
; COEXEC-NEXT: ds_load_tr16_b128 v[0:3], v72
; COEXEC-NEXT: ds_load_tr16_b128 v[4:7], v72 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v72 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v72 offset:192
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[8:15], v[24:31], v[48:55]
-; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v72 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v72 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v72 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v72 offset:192
+; COEXEC-NEXT: s_wait_dscnt 0x4
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[16:23], v[24:31], v[48:55]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[16:23], v[24:31], v[48:55]
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v72 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v72 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v72 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v72 offset:448
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[64:71], v[64:71], v[40:47]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[40:47], v[64:71], v[64:71], v[40:47]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[64:71], v[64:71], v[32:39]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[64:71], v[64:71], v[32:39]
; COEXEC-NEXT: s_cbranch_scc1 .LBB3_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_dscnt 0x7
-; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
@@ -1248,12 +1243,12 @@ define amdgpu_kernel void @ds_wmma_no_block_carried(ptr addrspace(3) %base, ptr
; COEXEC-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; COEXEC-NEXT: s_load_b32 s1, s[4:5], 0x10 nv
; COEXEC-NEXT: s_mov_b32 s9, s8
+; COEXEC-NEXT: s_mov_b32 s15, s8
; COEXEC-NEXT: s_mov_b32 s10, s8
; COEXEC-NEXT: s_mov_b32 s11, s8
; COEXEC-NEXT: s_mov_b32 s12, s8
; COEXEC-NEXT: s_mov_b32 s13, s8
; COEXEC-NEXT: s_mov_b32 s14, s8
-; COEXEC-NEXT: s_mov_b32 s15, s8
; COEXEC-NEXT: v_mov_b64_e32 v[102:103], s[14:15]
; COEXEC-NEXT: v_mov_b64_e32 v[100:101], s[12:13]
; COEXEC-NEXT: v_mov_b64_e32 v[98:99], s[10:11]
@@ -1264,54 +1259,53 @@ define amdgpu_kernel void @ds_wmma_no_block_carried(ptr addrspace(3) %base, ptr
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
; COEXEC-NEXT: ds_load_tr16_b128 v[0:3], v60
; COEXEC-NEXT: ds_load_tr16_b128 v[4:7], v60 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v60 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v60 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v60 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v60 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v60 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v60 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v60 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v60 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[12:15], v60 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[8:11], v60 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v60 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v60 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v60 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v60 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v60 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v60 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:832
; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v60 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v60 offset:960
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
; COEXEC-NEXT: s_wait_dscnt 0xe
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[0:7], v[96:103], 0
; COEXEC-NEXT: s_wait_dscnt 0xc
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[8:15], v[96:103], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[8:15], v[96:103], v[88:95]
; COEXEC-NEXT: s_wait_dscnt 0xa
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[16:23], v[96:103], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[16:23], v[96:103], 0
; COEXEC-NEXT: s_wait_dscnt 0x8
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[24:31], v[96:103], 0
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[24:31], v[96:103], v[80:87]
; COEXEC-NEXT: s_wait_dscnt 0x6
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[32:39], v[96:103], v[88:95]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[32:39], v[96:103], 0
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[40:47], v[96:103], v[80:87]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[40:47], v[96:103], v[72:79]
; COEXEC-NEXT: s_wait_dscnt 0x2
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[48:55], v[96:103], v[72:79]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[48:55], v[96:103], 0
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[56:63], v[96:103], v[64:71]
; COEXEC-NEXT: .LBB4_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[0:7], v[32:39], v[88:95]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[0:7], v[8:15], v[88:95]
; COEXEC-NEXT: s_and_b32 s1, s0, exec_lo
; COEXEC-NEXT: s_cselect_b32 s1, 1, 0
; COEXEC-NEXT: s_cmp_lg_u32 s1, 1
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[8:15], v[40:47], v[80:87]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[16:23], v[48:55], v[72:79]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[24:31], v[56:63], v[64:71]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[0:7], v[32:39], v[88:95]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[8:15], v[40:47], v[80:87]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[16:23], v[48:55], v[72:79]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[24:31], v[56:63], v[64:71]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[88:95], v[0:7], v[8:15], v[88:95]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[16:23], v[24:31], v[80:87]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[80:87], v[16:23], v[24:31], v[80:87]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[32:39], v[40:47], v[72:79]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[72:79], v[32:39], v[40:47], v[72:79]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[48:55], v[56:63], v[64:71]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[64:71], v[48:55], v[56:63], v[64:71]
; COEXEC-NEXT: s_cbranch_scc1 .LBB4_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
index e7ddfc73a8d60f..ec62735ec3a86a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
@@ -146,9 +146,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_cluster(ptr
; COEXEC-NEXT: v_swmmac_f16_16x16x32_f16 v[40:43], v[20:23], v[0:7], v48
; COEXEC-NEXT: v_swmmac_f16_16x16x32_f16 v[36:39], v[16:19], v[0:7], v48
; COEXEC-NEXT: ds_store_b128 v50, v[28:31]
+; COEXEC-NEXT: ds_store_b128 v49, v[40:43] offset:1536
; COEXEC-NEXT: ds_store_b128 v49, v[32:35] offset:512
; COEXEC-NEXT: ds_store_b128 v49, v[44:47] offset:1024
-; COEXEC-NEXT: ds_store_b128 v49, v[40:43] offset:1536
; COEXEC-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; COEXEC-NEXT: ds_store_b128 v49, v[36:39] offset:2048
; COEXEC-NEXT: ; sched_group_barrier mask(0x00000200) size(5) SyncID(0)
More information about the llvm-commits
mailing list