[llvm] [AMDGPU] DAG Mutation to solve load bunching in outer product matrix multiplications (PR #203095)
Axel Sorenson via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 15 00:48:46 PDT 2026
https://github.com/axelcool1234 updated https://github.com/llvm/llvm-project/pull/203095
>From a26e9913e052b09a5e9712a853b9d40bc22e9802 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 4 Jun 2026 23:36:28 +0000
Subject: [PATCH 1/9] DAG Mutation to solve ds_load bunching in outer product
matmuls
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 119 ++++++++++++++++++
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h | 24 ++++
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
4 files changed, 146 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 65317016c6390..c4c4977218746 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -39,6 +39,7 @@
#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUUnifyDivergentExitNodes.h"
#include "AMDGPUWaitSGPRHazards.h"
+#include "AMDGPUWMMASchedule.h"
#include "GCNDPPCombine.h"
#include "GCNIterativeScheduler.h"
#include "GCNNSAReassign.h"
@@ -759,6 +760,7 @@ createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));
DAG->addMutation(createAMDGPUHazardLatencyDAGMutation(C->MF));
+ DAG->addMutation(createAMDGPUWMMAScheduleDAGMutation(C->MF));
return DAG;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
new file mode 100644
index 0000000000000..e367ad9a22659
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -0,0 +1,119 @@
+//===--- AMDGPUWMMASchedule.cpp - AMDGPU WMMA Schedule Adjustment ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This file contains a DAG scheduling mutation to add additional
+/// edges between ds_load instructions and wmma instructions that
+/// occur a certain amount away from the actual wmma consumer of
+/// said ds_load. This forces the ds_load to properly prefetch
+/// and prevent early bunching of ds_loads that then lead to long
+/// stalls.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUWMMASchedule.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/Support/Debug.h"
+#include <optional>
+#define DEBUG_TYPE "amdgpu-wmma-sched"
+
+using namespace llvm;
+
+namespace {
+
+class WMMASchedule : public ScheduleDAGMutation {
+private:
+ const GCNSubtarget &ST;
+ const SIRegisterInfo &TRI;
+ const MachineRegisterInfo &MRI;
+
+public:
+ WMMASchedule(MachineFunction *MF)
+ : ST(MF->getSubtarget<GCNSubtarget>()), TRI(*ST.getRegisterInfo()),
+ MRI(MF->getRegInfo()) {}
+ void apply(ScheduleDAGInstrs *DAG) override;
+};
+
+void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
+ if (!ST.hasGFX1250Insts()) return;
+ const TargetSchedModel *SM = DAG->getSchedModel();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size() << "SUnits\n");
+
+ SmallVector<SUnit*> Loads;
+ MapVector<SUnit*, unsigned> Wmmas;
+
+ std::optional<unsigned> LoadLatency = std::nullopt;
+ std::optional<unsigned> WmmaLatency = std::nullopt;
+
+ // Gather all WMMAs and DS_LOADs
+ for(auto &SU : DAG->SUnits) {
+ MachineInstr* MI = SU.getInstr();
+ if(!MI) continue;
+
+ // Gather WMMAs
+ if(TII->isMFMAorWMMA(*MI)) {
+ if(WmmaLatency == std::nullopt) WmmaLatency = SM->computeInstrLatency(MI);
+ Wmmas.insert({&SU, Wmmas.size()});
+ continue;
+ }
+
+ // Gather DS_LOADs
+ if(TII->isDS(*MI) && MI->mayLoad()) {
+ if(LoadLatency == std::nullopt) LoadLatency = SM->computeInstrLatency(MI);
+ Loads.push_back(&SU);
+ }
+ }
+
+ // Calculate how many WMMAs away from consuming WMMA a load must be
+ // before it will certainly ready for consumer
+ unsigned Dist;
+ if(LoadLatency && WmmaLatency) {
+ Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1) Dist = 1;
+ } else {
+ return; // Either missing Wmmas or Loads. No point continuing.
+ }
+ LLVM_DEBUG(dbgs() << "Dist " << Dist << "\n");
+
+ // For every load, determine earliest WMMA reliant on it,
+ // and add an anchor.
+ for(SUnit* L : Loads){
+ SUnit* Earliest = nullptr;
+ unsigned EarliestPos = UINT_MAX;
+ for(const SDep& D : L->Succs){
+ if(D.getKind() != SDep::Data) continue;
+ SUnit* S = D.getSUnit();
+ auto *It = Wmmas.find(S);
+ if(It == Wmmas.end()) continue;
+ if(It->second < EarliestPos) {
+ EarliestPos = It->second;
+ Earliest = S;
+ }
+ };
+ LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
+ << (Earliest ? (int)Earliest->NodeNum : -1)
+ << " (pos " << EarliestPos << ")\n");
+ if(Earliest && EarliestPos >= Dist) {
+ LLVM_DEBUG(dbgs() << "Window Created!\n");
+ SUnit* Anchor = Wmmas.begin()[EarliestPos - Dist].first;
+ bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
+ LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU" << Anchor->NodeNum
+ << (Ok ? "\n" : " (REJECTED: cycle)\n"));
+ };
+ };
+}
+
+} // end namespace
+
+std::unique_ptr<ScheduleDAGMutation>
+llvm::createAMDGPUWMMAScheduleDAGMutation(MachineFunction *MF) {
+ return std::make_unique<WMMASchedule>(MF);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
new file mode 100644
index 0000000000000..781f4b3b1ac86
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
@@ -0,0 +1,24 @@
+//===- AMDGPUWMMASchedule.h - WMMA Schedule Adjustment ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
+
+#include "llvm/CodeGen/ScheduleDAGMutation.h"
+#include <memory>
+
+namespace llvm {
+
+class MachineFunction;
+
+std::unique_ptr<ScheduleDAGMutation>
+createAMDGPUWMMAScheduleDAGMutation(MachineFunction *MF);
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index ae8f1c0fad5ba..82f129b74076c 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -122,6 +122,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUTargetTransformInfo.cpp
AMDGPUWaitcntUtils.cpp
AMDGPUWaitSGPRHazards.cpp
+ AMDGPUWMMASchedule.cpp
AMDGPUUnifyDivergentExitNodes.cpp
R600MachineCFGStructurizer.cpp
GCNCreateVOPD.cpp
>From 7e16bae438d4ac052f5b57b4000b2a0579c05778 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Wed, 10 Jun 2026 20:23:37 +0000
Subject: [PATCH 2/9] Clang format
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +-
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 69 +++++++++++--------
2 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index c4c4977218746..edbc8cafbde54 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -38,8 +38,8 @@
#include "AMDGPUTargetObjectFile.h"
#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUUnifyDivergentExitNodes.h"
-#include "AMDGPUWaitSGPRHazards.h"
#include "AMDGPUWMMASchedule.h"
+#include "AMDGPUWaitSGPRHazards.h"
#include "GCNDPPCombine.h"
#include "GCNIterativeScheduler.h"
#include "GCNNSAReassign.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index e367ad9a22659..d87326ba00b49 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -42,42 +42,48 @@ class WMMASchedule : public ScheduleDAGMutation {
};
void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
- if (!ST.hasGFX1250Insts()) return;
+ if (!ST.hasGFX1250Insts())
+ return;
const TargetSchedModel *SM = DAG->getSchedModel();
const SIInstrInfo *TII = ST.getInstrInfo();
- LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size() << "SUnits\n");
+ LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size()
+ << "SUnits\n");
- SmallVector<SUnit*> Loads;
- MapVector<SUnit*, unsigned> Wmmas;
+ SmallVector<SUnit *> Loads;
+ MapVector<SUnit *, unsigned> Wmmas;
std::optional<unsigned> LoadLatency = std::nullopt;
std::optional<unsigned> WmmaLatency = std::nullopt;
-
+
// Gather all WMMAs and DS_LOADs
- for(auto &SU : DAG->SUnits) {
- MachineInstr* MI = SU.getInstr();
- if(!MI) continue;
+ for (auto &SU : DAG->SUnits) {
+ MachineInstr *MI = SU.getInstr();
+ if (!MI)
+ continue;
// Gather WMMAs
- if(TII->isMFMAorWMMA(*MI)) {
- if(WmmaLatency == std::nullopt) WmmaLatency = SM->computeInstrLatency(MI);
+ if (TII->isMFMAorWMMA(*MI)) {
+ if (WmmaLatency == std::nullopt)
+ WmmaLatency = SM->computeInstrLatency(MI);
Wmmas.insert({&SU, Wmmas.size()});
continue;
}
// Gather DS_LOADs
- if(TII->isDS(*MI) && MI->mayLoad()) {
- if(LoadLatency == std::nullopt) LoadLatency = SM->computeInstrLatency(MI);
+ if (TII->isDS(*MI) && MI->mayLoad()) {
+ if (LoadLatency == std::nullopt)
+ LoadLatency = SM->computeInstrLatency(MI);
Loads.push_back(&SU);
}
}
- // Calculate how many WMMAs away from consuming WMMA a load must be
+ // Calculate how many WMMAs away from consuming WMMA a load must be
// before it will certainly ready for consumer
unsigned Dist;
- if(LoadLatency && WmmaLatency) {
- Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
- if (Dist < 1) Dist = 1;
+ if (LoadLatency && WmmaLatency) {
+ Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1)
+ Dist = 1;
} else {
return; // Either missing Wmmas or Loads. No point continuing.
}
@@ -85,28 +91,31 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// For every load, determine earliest WMMA reliant on it,
// and add an anchor.
- for(SUnit* L : Loads){
- SUnit* Earliest = nullptr;
+ for (SUnit *L : Loads) {
+ SUnit *Earliest = nullptr;
unsigned EarliestPos = UINT_MAX;
- for(const SDep& D : L->Succs){
- if(D.getKind() != SDep::Data) continue;
- SUnit* S = D.getSUnit();
+ for (const SDep &D : L->Succs) {
+ if (D.getKind() != SDep::Data)
+ continue;
+ SUnit *S = D.getSUnit();
auto *It = Wmmas.find(S);
- if(It == Wmmas.end()) continue;
- if(It->second < EarliestPos) {
- EarliestPos = It->second;
+ if (It == Wmmas.end())
+ continue;
+ if (It->second < EarliestPos) {
+ EarliestPos = It->second;
Earliest = S;
}
};
LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
- << (Earliest ? (int)Earliest->NodeNum : -1)
- << " (pos " << EarliestPos << ")\n");
- if(Earliest && EarliestPos >= Dist) {
+ << (Earliest ? (int)Earliest->NodeNum : -1) << " (pos "
+ << EarliestPos << ")\n");
+ if (Earliest && EarliestPos >= Dist) {
LLVM_DEBUG(dbgs() << "Window Created!\n");
- SUnit* Anchor = Wmmas.begin()[EarliestPos - Dist].first;
+ SUnit *Anchor = Wmmas.begin()[EarliestPos - Dist].first;
bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
- LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU" << Anchor->NodeNum
- << (Ok ? "\n" : " (REJECTED: cycle)\n"));
+ LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU"
+ << Anchor->NodeNum
+ << (Ok ? "\n" : " (REJECTED: cycle)\n"));
};
};
}
>From d0c4dade1e4280a67bd43f84286709c4ef525846 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Wed, 10 Jun 2026 23:25:51 +0000
Subject: [PATCH 3/9] Maximum and minimum distance
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 125 +++++++++++-------
1 file changed, 76 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index d87326ba00b49..fba0109a2fcca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -28,6 +28,14 @@ using namespace llvm;
namespace {
+// A ds_load plus the program order positions (among WMMAs) of its
+// earliest and latest WMMA consumer.
+struct LoadInfo {
+ SUnit *SU;
+ unsigned MinPos = UINT_MAX; // earliest consumer (UINT_MAX means none in this region)
+ unsigned MaxPos = 0; // latest consumer
+};
+
class WMMASchedule : public ScheduleDAGMutation {
private:
const GCNSubtarget &ST;
@@ -46,24 +54,20 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
return;
const TargetSchedModel *SM = DAG->getSchedModel();
const SIInstrInfo *TII = ST.getInstrInfo();
- LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size()
- << "SUnits\n");
-
- SmallVector<SUnit *> Loads;
- MapVector<SUnit *, unsigned> Wmmas;
- std::optional<unsigned> LoadLatency = std::nullopt;
- std::optional<unsigned> WmmaLatency = std::nullopt;
+ // Gather WMMAs (numbered in program order) and ds_loads.
+ MapVector<SUnit *, unsigned> Wmmas; // WMMA SUnit and its program position relative to one another
+ SmallVector<LoadInfo> Loads;
+ std::optional<unsigned> LoadLatency, WmmaLatency;
- // Gather all WMMAs and DS_LOADs
- for (auto &SU : DAG->SUnits) {
+ for (SUnit &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
if (!MI)
continue;
// Gather WMMAs
if (TII->isMFMAorWMMA(*MI)) {
- if (WmmaLatency == std::nullopt)
+ if (!WmmaLatency)
WmmaLatency = SM->computeInstrLatency(MI);
Wmmas.insert({&SU, Wmmas.size()});
continue;
@@ -71,53 +75,76 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather DS_LOADs
if (TII->isDS(*MI) && MI->mayLoad()) {
- if (LoadLatency == std::nullopt)
+ if (!LoadLatency)
LoadLatency = SM->computeInstrLatency(MI);
- Loads.push_back(&SU);
+ Loads.push_back({&SU});
}
}
- // Calculate how many WMMAs away from consuming WMMA a load must be
- // before it will certainly ready for consumer
- unsigned Dist;
- if (LoadLatency && WmmaLatency) {
- Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
- if (Dist < 1)
- Dist = 1;
- } else {
- return; // Either missing Wmmas or Loads. No point continuing.
- }
- LLVM_DEBUG(dbgs() << "Dist " << Dist << "\n");
-
- // For every load, determine earliest WMMA reliant on it,
- // and add an anchor.
- for (SUnit *L : Loads) {
- SUnit *Earliest = nullptr;
- unsigned EarliestPos = UINT_MAX;
- for (const SDep &D : L->Succs) {
+ // The following means the DAG Mutation cannot do anything useful.
+ if (!LoadLatency || !WmmaLatency || Wmmas.empty())
+ return;
+
+ // The number of WMMAs that elapse during one load's latency
+ unsigned Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1)
+ Dist = 1;
+
+ // For each load, find the earliest and latest consuming WMMA positions.
+ for (LoadInfo& LI : Loads) {
+ for (const SDep &D : LI.SU->Succs) {
if (D.getKind() != SDep::Data)
continue;
- SUnit *S = D.getSUnit();
- auto *It = Wmmas.find(S);
+ auto *It = Wmmas.find(D.getSUnit());
+ // Check to see if successor is a WMMA
if (It == Wmmas.end())
continue;
- if (It->second < EarliestPos) {
- EarliestPos = It->second;
- Earliest = S;
- }
- };
- LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
- << (Earliest ? (int)Earliest->NodeNum : -1) << " (pos "
- << EarliestPos << ")\n");
- if (Earliest && EarliestPos >= Dist) {
- LLVM_DEBUG(dbgs() << "Window Created!\n");
- SUnit *Anchor = Wmmas.begin()[EarliestPos - Dist].first;
- bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
- LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU"
- << Anchor->NodeNum
- << (Ok ? "\n" : " (REJECTED: cycle)\n"));
- };
- };
+ LI.MinPos = std::min(LI.MinPos, It->second);
+ LI.MaxPos = std::max(LI.MaxPos, It->second);
+ }
+ }
+
+ // MaxPos in order
+ std::vector<bool> Present(Wmmas.size(), false);
+ for (const LoadInfo &LI : Loads)
+ // Check if there's a consumer for this load in the region
+ if (LI.MinPos != UINT_MAX)
+ Present[LI.MaxPos] = true;
+
+ // Latest position that's <= position Pos at which some load's register frees
+ // A load's register becomes free at its MaxPos, which is its last WMMA consumer.
+ std::vector<std::optional<unsigned>> DeadBy(Wmmas.size(), std::nullopt);
+ std::optional<unsigned> Latest;
+ for (unsigned Pos = 0; Pos < Wmmas.size(); ++Pos) {
+ if (Present[Pos])
+ Latest = Pos;
+ DeadBy[Pos] = Latest;
+ }
+
+ // Create the edges that constrain where the ds_loads can be placed
+ // minimum distance of a load is LI.MinPos - Dist
+ // maximum distance of a load is DeadBy[LI.MinPos - Dist]
+ for (const LoadInfo &LI : Loads) {
+ if (LI.MinPos == UINT_MAX || LI.MinPos < Dist)
+ continue;
+
+ // Minimum distance edge
+ // Load scheduled before this point
+ unsigned MinDist = LI.MinPos - Dist;
+ bool MinSuccess = DAG->addEdge(Wmmas.begin()[MinDist].first, SDep(LI.SU, SDep::Artificial));
+
+ // Maximum distance edge
+ // Load scheduled after this point
+ bool MaxSuccess = true;
+ std::optional<unsigned> MaxDist = DeadBy[MinDist];
+ if (MaxDist && *MaxDist < MinDist)
+ MaxSuccess = DAG->addEdge(LI.SU, SDep(Wmmas.begin()[*MaxDist].first, SDep::Artificial));
+
+ LLVM_DEBUG(dbgs() << "load SU" << LI.SU->NodeNum << " Win=[" << (MaxDist ? *MaxDist : -1) << ","
+ << MinDist << "] MinPos=" << LI.MinPos << " MaxPos=" << LI.MaxPos
+ << (MinSuccess && MaxSuccess ? "\n" : " (edge REJECTED: cycle)\n"));
+ }
+
}
} // end namespace
>From f53deeaa2e17de2227df87d5c7ea441467d5a9c8 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 11 Jun 2026 19:34:03 +0000
Subject: [PATCH 4/9] MIR test
---
.../AMDGPU/sched-wmma-ds-load-window.mir | 376 ++++++++++++++++++
1 file changed, 376 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
diff --git a/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir b/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
new file mode 100644
index 0000000000000..6435dc89889e3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
@@ -0,0 +1,376 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+#
+# Test for the WMMASchedule DAG mutation on an 8x8 outer product tile
+# (64 DS_READ_B128, 8 A/B fragments, 64 V_WMMA). Without the mutation the
+# scheduler schedules all 64 loads at the beginning of the block; the mutation
+# debunches these loads.
+#
+# XFAIL: *
+
+--- |
+ target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+ target triple = "amdgcn"
+ define amdgpu_kernel void @wmma_op() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx1250" "amdgpu-flat-work-group-size"="1,128" "amdgpu-waves-per-eu"="1,1" }
+...
+---
+name: wmma_op
+alignment: 4
+tracksRegLiveness: true
+isSSA: false
+noPhis: true
+machineFunctionInfo:
+ isEntryFunction: true
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+ occupancy: 1
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: wmma_op
+ ; CHECK: V_WMMA
+ ; CHECK-NEXT: V_WMMA
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: V_WMMA
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: V_WMMA
+ successors: %bb.1(0x80000000)
+ %ptra:vgpr_32 = IMPLICIT_DEF
+ %ptrb:vgpr_32 = IMPLICIT_DEF
+ %scale:vreg_64_lo256_align2 = IMPLICIT_DEF
+ %vo:vgpr_32 = IMPLICIT_DEF
+ %rsrc:sgpr_128 = IMPLICIT_DEF
+ %500:vreg_256_align2 = IMPLICIT_DEF
+ %501:vreg_256_align2 = IMPLICIT_DEF
+ %502:vreg_256_align2 = IMPLICIT_DEF
+ %503:vreg_256_align2 = IMPLICIT_DEF
+ %504:vreg_256_align2 = IMPLICIT_DEF
+ %505:vreg_256_align2 = IMPLICIT_DEF
+ %506:vreg_256_align2 = IMPLICIT_DEF
+ %507:vreg_256_align2 = IMPLICIT_DEF
+ %508:vreg_256_align2 = IMPLICIT_DEF
+ %509:vreg_256_align2 = IMPLICIT_DEF
+ %510:vreg_256_align2 = IMPLICIT_DEF
+ %511:vreg_256_align2 = IMPLICIT_DEF
+ %512:vreg_256_align2 = IMPLICIT_DEF
+ %513:vreg_256_align2 = IMPLICIT_DEF
+ %514:vreg_256_align2 = IMPLICIT_DEF
+ %515:vreg_256_align2 = IMPLICIT_DEF
+ %516:vreg_256_align2 = IMPLICIT_DEF
+ %517:vreg_256_align2 = IMPLICIT_DEF
+ %518:vreg_256_align2 = IMPLICIT_DEF
+ %519:vreg_256_align2 = IMPLICIT_DEF
+ %520:vreg_256_align2 = IMPLICIT_DEF
+ %521:vreg_256_align2 = IMPLICIT_DEF
+ %522:vreg_256_align2 = IMPLICIT_DEF
+ %523:vreg_256_align2 = IMPLICIT_DEF
+ %524:vreg_256_align2 = IMPLICIT_DEF
+ %525:vreg_256_align2 = IMPLICIT_DEF
+ %526:vreg_256_align2 = IMPLICIT_DEF
+ %527:vreg_256_align2 = IMPLICIT_DEF
+ %528:vreg_256_align2 = IMPLICIT_DEF
+ %529:vreg_256_align2 = IMPLICIT_DEF
+ %530:vreg_256_align2 = IMPLICIT_DEF
+ %531:vreg_256_align2 = IMPLICIT_DEF
+ %532:vreg_256_align2 = IMPLICIT_DEF
+ %533:vreg_256_align2 = IMPLICIT_DEF
+ %534:vreg_256_align2 = IMPLICIT_DEF
+ %535:vreg_256_align2 = IMPLICIT_DEF
+ %536:vreg_256_align2 = IMPLICIT_DEF
+ %537:vreg_256_align2 = IMPLICIT_DEF
+ %538:vreg_256_align2 = IMPLICIT_DEF
+ %539:vreg_256_align2 = IMPLICIT_DEF
+ %540:vreg_256_align2 = IMPLICIT_DEF
+ %541:vreg_256_align2 = IMPLICIT_DEF
+ %542:vreg_256_align2 = IMPLICIT_DEF
+ %543:vreg_256_align2 = IMPLICIT_DEF
+ %544:vreg_256_align2 = IMPLICIT_DEF
+ %545:vreg_256_align2 = IMPLICIT_DEF
+ %546:vreg_256_align2 = IMPLICIT_DEF
+ %547:vreg_256_align2 = IMPLICIT_DEF
+ %548:vreg_256_align2 = IMPLICIT_DEF
+ %549:vreg_256_align2 = IMPLICIT_DEF
+ %550:vreg_256_align2 = IMPLICIT_DEF
+ %551:vreg_256_align2 = IMPLICIT_DEF
+ %552:vreg_256_align2 = IMPLICIT_DEF
+ %553:vreg_256_align2 = IMPLICIT_DEF
+ %554:vreg_256_align2 = IMPLICIT_DEF
+ %555:vreg_256_align2 = IMPLICIT_DEF
+ %556:vreg_256_align2 = IMPLICIT_DEF
+ %557:vreg_256_align2 = IMPLICIT_DEF
+ %558:vreg_256_align2 = IMPLICIT_DEF
+ %559:vreg_256_align2 = IMPLICIT_DEF
+ %560:vreg_256_align2 = IMPLICIT_DEF
+ %561:vreg_256_align2 = IMPLICIT_DEF
+ %562:vreg_256_align2 = IMPLICIT_DEF
+ %563:vreg_256_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ undef %300.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %301.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %302.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %303.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %304.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %305.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %306.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %307.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %400.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %401.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %402.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %403.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %404.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %405.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %406.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %407.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ early-clobber %500:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %400, 8, %500, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %501:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %401, 8, %501, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %502:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %402, 8, %502, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %503:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %403, 8, %503, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %504:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %404, 8, %504, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %505:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %405, 8, %505, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %506:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %406, 8, %506, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %507:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %407, 8, %507, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %508:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %400, 8, %508, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %509:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %401, 8, %509, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %510:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %402, 8, %510, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %511:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %403, 8, %511, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %512:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %404, 8, %512, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %513:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %405, 8, %513, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %514:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %406, 8, %514, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %515:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %407, 8, %515, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %516:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %400, 8, %516, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %517:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %401, 8, %517, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %518:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %402, 8, %518, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %519:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %403, 8, %519, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %520:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %404, 8, %520, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %521:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %405, 8, %521, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %522:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %406, 8, %522, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %523:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %407, 8, %523, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %524:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %400, 8, %524, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %525:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %401, 8, %525, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %526:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %402, 8, %526, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %527:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %403, 8, %527, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %528:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %404, 8, %528, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %529:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %405, 8, %529, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %530:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %406, 8, %530, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %531:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %407, 8, %531, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %532:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %400, 8, %532, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %533:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %401, 8, %533, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %534:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %402, 8, %534, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %535:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %403, 8, %535, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %536:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %404, 8, %536, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %537:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %405, 8, %537, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %538:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %406, 8, %538, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %539:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %407, 8, %539, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %540:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %400, 8, %540, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %541:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %401, 8, %541, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %542:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %402, 8, %542, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %543:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %403, 8, %543, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %544:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %404, 8, %544, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %545:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %405, 8, %545, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %546:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %406, 8, %546, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %547:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %407, 8, %547, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %548:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %400, 8, %548, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %549:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %401, 8, %549, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %550:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %402, 8, %550, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %551:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %403, 8, %551, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %552:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %404, 8, %552, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %553:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %405, 8, %553, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %554:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %406, 8, %554, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %555:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %407, 8, %555, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %556:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %400, 8, %556, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %557:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %401, 8, %557, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %558:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %402, 8, %558, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %559:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %403, 8, %559, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %560:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %404, 8, %560, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %561:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %405, 8, %561, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %562:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %406, 8, %562, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %563:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %407, 8, %563, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %500.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %500.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %501.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %501.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %502.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %502.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %503.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %503.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %504.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %504.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %505.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %505.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %506.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %506.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %507.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %507.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %508.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %508.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %509.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %509.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %510.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %510.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %511.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %511.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %512.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %512.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %513.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %513.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %514.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %514.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %515.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %515.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %516.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %516.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %517.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %517.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %518.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %518.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %519.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %519.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %520.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %520.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %521.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %521.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %522.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %522.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %523.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %523.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %524.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %524.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %525.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %525.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %526.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %526.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %527.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %527.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %528.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %528.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %529.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %529.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %530.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %530.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %531.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %531.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %532.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %532.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %533.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %533.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %534.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %534.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %535.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %535.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %536.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %536.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %537.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %537.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %538.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %538.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %539.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %539.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %540.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %540.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %541.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %541.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %542.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %542.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %543.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %543.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %544.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %544.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %545.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %545.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %546.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %546.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %547.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %547.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %548.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %548.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %549.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %549.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %550.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %550.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %551.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %551.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %552.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %552.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %553.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %553.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %554.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %554.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %555.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %555.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %556.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %556.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %557.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %557.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %558.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %558.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %559.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %559.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %560.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %560.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %561.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %561.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %562.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %562.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %563.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %563.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
>From 2f84bfee8b885676ac43de224e5bee00b22f2cf2 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 11 Jun 2026 21:08:50 +0000
Subject: [PATCH 5/9] Ordering WMMAs
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index fba0109a2fcca..897b1c20642f6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -90,6 +90,16 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
if (Dist < 1)
Dist = 1;
+ // Ensure ordering of WMMAs.
+ auto [PrevSU, _] = *Wmmas.begin();
+ for (auto *It = std::next(Wmmas.begin()); It != Wmmas.end(); ++It) {
+ auto [SU, _] = *It;
+ bool Success = DAG->addEdge(SU, SDep(PrevSU, SDep::Artificial));
+ LLVM_DEBUG(dbgs() << "wmma SU" << SU->NodeNum << " <- after wmma SU"
+ << PrevSU->NodeNum << (Success ? "\n" : " FAIL (cycle)\n"));
+ PrevSU = SU;
+ }
+
// For each load, find the earliest and latest consuming WMMA positions.
for (LoadInfo& LI : Loads) {
for (const SDep &D : LI.SU->Succs) {
>From 128bbec30bb5cf184185a98ff39f9ed7ebdd561f Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 11 Jun 2026 22:12:31 +0000
Subject: [PATCH 6/9] Bandwidth bounded ds_loads (keeping ds_loads from being
too close)
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 51 +++++++++++++++----
1 file changed, 41 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index 897b1c20642f6..809704a1ead73 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -34,6 +34,8 @@ struct LoadInfo {
SUnit *SU;
unsigned MinPos = UINT_MAX; // earliest consumer (UINT_MAX means none in this region)
unsigned MaxPos = 0; // latest consumer
+ long LatestCycle = 0; // Latest cycle this load can be scheduled for
+ bool BandwidthBound = false; // If bandwidth bound, it'll be placest at LatestCycle or earlier. Else, MaxPos.
};
class WMMASchedule : public ScheduleDAGMutation {
@@ -59,6 +61,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
MapVector<SUnit *, unsigned> Wmmas; // WMMA SUnit and its program position relative to one another
SmallVector<LoadInfo> Loads;
std::optional<unsigned> LoadLatency, WmmaLatency;
+ std::optional<double> LDSBandwidth;
for (SUnit &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
@@ -75,21 +78,18 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather DS_LOADs
if (TII->isDS(*MI) && MI->mayLoad()) {
- if (!LoadLatency)
- LoadLatency = SM->computeInstrLatency(MI);
+ if (!LoadLatency) {
+ LoadLatency = SM->computeInstrLatency(MI); // TODO: Hardcode this.
+ LDSBandwidth = std::ceil(SM->computeReciprocalThroughput(MI)); // TODO: Possibly hardcode this?
+ }
Loads.push_back({&SU});
}
}
// The following means the DAG Mutation cannot do anything useful.
- if (!LoadLatency || !WmmaLatency || Wmmas.empty())
+ if (!LoadLatency || !LDSBandwidth || !WmmaLatency || Wmmas.empty())
return;
- // The number of WMMAs that elapse during one load's latency
- unsigned Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
- if (Dist < 1)
- Dist = 1;
-
// Ensure ordering of WMMAs.
auto [PrevSU, _] = *Wmmas.begin();
for (auto *It = std::next(Wmmas.begin()); It != Wmmas.end(); ++It) {
@@ -114,6 +114,11 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
}
}
+ // Order the Loads
+ llvm::stable_sort(Loads, [](const LoadInfo &A, const LoadInfo &B) {
+ return A.MinPos < B.MinPos;
+ });
+
// MaxPos in order
std::vector<bool> Present(Wmmas.size(), false);
for (const LoadInfo &LI : Loads)
@@ -131,17 +136,43 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
DeadBy[Pos] = Latest;
}
+ // For each load, determine if it needs to be bandwidth bound to prevent
+ // being too close to other loads.
+ for (LoadInfo& LI : Loads) {
+ if (LI.MinPos != UINT_MAX)
+ // Same thing as MinPos, but in cycles
+ LI.LatestCycle = (long)(LI.MinPos) * (*WmmaLatency) - (long)(*LoadLatency);
+ }
+ long PrevLatest = LONG_MAX;
+ for(int Pos = static_cast<int>(Loads.size()) - 1; Pos >= 0; --Pos) {
+ LoadInfo& LI = Loads[Pos];
+ if (LI.MinPos == UINT_MAX)
+ continue;
+ long Spaced = PrevLatest - (long)(*LDSBandwidth);
+ // Clamp to Spaced if the LatestCycle encroaches too close to another load
+ if (Spaced < LI.LatestCycle) {
+ LI.LatestCycle = Spaced;
+ LI.BandwidthBound = true;
+ }
+ PrevLatest = LI.LatestCycle;
+ }
+
// Create the edges that constrain where the ds_loads can be placed
// minimum distance of a load is LI.MinPos - Dist
// maximum distance of a load is DeadBy[LI.MinPos - Dist]
for (const LoadInfo &LI : Loads) {
- if (LI.MinPos == UINT_MAX || LI.MinPos < Dist)
+ if (LI.MinPos == UINT_MAX)
continue;
// Minimum distance edge
// Load scheduled before this point
- unsigned MinDist = LI.MinPos - Dist;
+ long MinDist = LI.LatestCycle / (long)(*WmmaLatency); // Go from cycle back to WMMA position (floor division)
+ if (MinDist < 0)
+ continue;
bool MinSuccess = DAG->addEdge(Wmmas.begin()[MinDist].first, SDep(LI.SU, SDep::Artificial));
+ LLVM_DEBUG(dbgs() << (LI.BandwidthBound ? "[bw] " : "[lat] ")
+ << "load SU" << LI.SU->NodeNum << " before W[" << MinDist
+ << "] (MinPos=" << LI.MinPos << " LatestCycle=" << LI.LatestCycle << ")\n");
// Maximum distance edge
// Load scheduled after this point
>From 0a77220577d50a897b5d31279c35f0b06e84a806 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Fri, 12 Jun 2026 00:00:50 +0000
Subject: [PATCH 7/9] DAG mutation now relies on latencies placed on edges.
Added ds_load->ds_load edges with hardcoded latency. Modified
ds_load->earliest wmma consumer edges to use hardcoded latency instead.
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 76 +++++++++----------
1 file changed, 37 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index 809704a1ead73..8729c117bf920 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -34,8 +34,6 @@ struct LoadInfo {
SUnit *SU;
unsigned MinPos = UINT_MAX; // earliest consumer (UINT_MAX means none in this region)
unsigned MaxPos = 0; // latest consumer
- long LatestCycle = 0; // Latest cycle this load can be scheduled for
- bool BandwidthBound = false; // If bandwidth bound, it'll be placest at LatestCycle or earlier. Else, MaxPos.
};
class WMMASchedule : public ScheduleDAGMutation {
@@ -60,7 +58,8 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather WMMAs (numbered in program order) and ds_loads.
MapVector<SUnit *, unsigned> Wmmas; // WMMA SUnit and its program position relative to one another
SmallVector<LoadInfo> Loads;
- std::optional<unsigned> LoadLatency, WmmaLatency;
+ std::optional<unsigned> LoadLatency = 64;
+ std::optional<unsigned> WmmaLatency;
std::optional<double> LDSBandwidth;
for (SUnit &SU : DAG->SUnits) {
@@ -78,10 +77,10 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather DS_LOADs
if (TII->isDS(*MI) && MI->mayLoad()) {
- if (!LoadLatency) {
- LoadLatency = SM->computeInstrLatency(MI); // TODO: Hardcode this.
- LDSBandwidth = std::ceil(SM->computeReciprocalThroughput(MI)); // TODO: Possibly hardcode this?
- }
+ if (!LoadLatency)
+ LoadLatency = SM->computeInstrLatency(MI);
+ if (!LDSBandwidth)
+ LDSBandwidth = std::ceil(SM->computeReciprocalThroughput(MI));
Loads.push_back({&SU});
}
}
@@ -90,7 +89,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
if (!LoadLatency || !LDSBandwidth || !WmmaLatency || Wmmas.empty())
return;
- // Ensure ordering of WMMAs.
+ // Order the WMMAs.
auto [PrevSU, _] = *Wmmas.begin();
for (auto *It = std::next(Wmmas.begin()); It != Wmmas.end(); ++It) {
auto [SU, _] = *It;
@@ -101,6 +100,8 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
}
// For each load, find the earliest and latest consuming WMMA positions.
+ // Additionally correct the latency of the ds_load -> earliest WMMA consumer
+ // data edge.
for (LoadInfo& LI : Loads) {
for (const SDep &D : LI.SU->Succs) {
if (D.getKind() != SDep::Data)
@@ -112,12 +113,35 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
LI.MinPos = std::min(LI.MinPos, It->second);
LI.MaxPos = std::max(LI.MaxPos, It->second);
}
+ if (LI.MinPos == UINT_MAX)
+ continue;
+ SUnit *EarliestConsumer = Wmmas.begin()[LI.MinPos].first;
+ // Correct latency of edges between ds_load and earliest WMMA consumer
+ for (SDep &S : LI.SU->Succs)
+ if (S.getSUnit() == EarliestConsumer && S.getKind() == SDep::Data)
+ S.setLatency(*LoadLatency);
+ for (SDep &P : EarliestConsumer->Preds)
+ if (P.getSUnit() == LI.SU && P.getKind() == SDep::Data)
+ P.setLatency(*LoadLatency);
+ EarliestConsumer->setDepthDirty();
+ LI.SU->setHeightDirty();
}
// Order the Loads
llvm::stable_sort(Loads, [](const LoadInfo &A, const LoadInfo &B) {
return A.MinPos < B.MinPos;
});
+ SUnit* Prev = nullptr;
+ for (LoadInfo &LI : Loads) {
+ if (LI.MinPos == UINT_MAX)
+ continue;
+ if (Prev) {
+ SDep D(Prev, SDep::Artificial);
+ D.setLatency(*LDSBandwidth);
+ DAG->addEdge(LI.SU, D);
+ }
+ Prev = LI.SU;
+ }
// MaxPos in order
std::vector<bool> Present(Wmmas.size(), false);
@@ -136,43 +160,17 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
DeadBy[Pos] = Latest;
}
- // For each load, determine if it needs to be bandwidth bound to prevent
- // being too close to other loads.
- for (LoadInfo& LI : Loads) {
- if (LI.MinPos != UINT_MAX)
- // Same thing as MinPos, but in cycles
- LI.LatestCycle = (long)(LI.MinPos) * (*WmmaLatency) - (long)(*LoadLatency);
- }
- long PrevLatest = LONG_MAX;
- for(int Pos = static_cast<int>(Loads.size()) - 1; Pos >= 0; --Pos) {
- LoadInfo& LI = Loads[Pos];
- if (LI.MinPos == UINT_MAX)
- continue;
- long Spaced = PrevLatest - (long)(*LDSBandwidth);
- // Clamp to Spaced if the LatestCycle encroaches too close to another load
- if (Spaced < LI.LatestCycle) {
- LI.LatestCycle = Spaced;
- LI.BandwidthBound = true;
- }
- PrevLatest = LI.LatestCycle;
- }
-
// Create the edges that constrain where the ds_loads can be placed
// minimum distance of a load is LI.MinPos - Dist
// maximum distance of a load is DeadBy[LI.MinPos - Dist]
+ unsigned Dist = std::max(1u, static_cast<unsigned>(std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency)));
for (const LoadInfo &LI : Loads) {
- if (LI.MinPos == UINT_MAX)
+ if (LI.MinPos == UINT_MAX || LI.MinPos < Dist)
continue;
// Minimum distance edge
- // Load scheduled before this point
- long MinDist = LI.LatestCycle / (long)(*WmmaLatency); // Go from cycle back to WMMA position (floor division)
- if (MinDist < 0)
- continue;
- bool MinSuccess = DAG->addEdge(Wmmas.begin()[MinDist].first, SDep(LI.SU, SDep::Artificial));
- LLVM_DEBUG(dbgs() << (LI.BandwidthBound ? "[bw] " : "[lat] ")
- << "load SU" << LI.SU->NodeNum << " before W[" << MinDist
- << "] (MinPos=" << LI.MinPos << " LatestCycle=" << LI.LatestCycle << ")\n");
+ // Load scheduled before this point (heuristically)
+ long MinDist = LI.MinPos - Dist;
// Maximum distance edge
// Load scheduled after this point
@@ -183,7 +181,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
LLVM_DEBUG(dbgs() << "load SU" << LI.SU->NodeNum << " Win=[" << (MaxDist ? *MaxDist : -1) << ","
<< MinDist << "] MinPos=" << LI.MinPos << " MaxPos=" << LI.MaxPos
- << (MinSuccess && MaxSuccess ? "\n" : " (edge REJECTED: cycle)\n"));
+ << (MaxSuccess ? "\n" : " (edge REJECTED: cycle)\n"));
}
}
>From 525293572c367f764fa82b41486a88d3ada28227 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Mon, 15 Jun 2026 02:42:33 +0000
Subject: [PATCH 8/9] Determine earliest point a ds_load should be loaded by
calculating overlapping live ranges
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 170 ++++++++++++------
1 file changed, 113 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index 8729c117bf920..85ea68f4ebf62 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -6,13 +6,24 @@
//
//===----------------------------------------------------------------------===//
//
-/// \file This file contains a DAG scheduling mutation to add additional
-/// edges between ds_load instructions and wmma instructions that
-/// occur a certain amount away from the actual wmma consumer of
-/// said ds_load. This forces the ds_load to properly prefetch
-/// and prevent early bunching of ds_loads that then lead to long
-/// stalls.
-//
+/// \file This file contains a DAG scheduling mutation that shapes how gfx1250
+/// ds_load (LDS) prefetches are placed relative to the WMMA instructions
+/// that consume them, to prevent the pre-RA scheduler from bunching all
+/// the loads at the head of the block (which forces the WMMAs behind
+/// long s_wait_dscnt stalls and inflates register pressure).
+///
+/// It does the following:
+/// - Order the WMMAs (WMMA -> WMMA edges added).
+/// - Order the ds_loads (ds_load -> ds_load edges added
+/// with latency attached to prevent them overhwelming
+/// the LDS bus and becoming memory bound).
+/// - Add WMMA -> ds_load edges to stop loads from being bunched at
+/// the start of the block
+/// - Build a live range histogram of the A/B operand fragments under
+/// an as late as possible schedule, recording the minimum VGPRs
+/// needed for such a schedule (so the WMMA -> ds_load edges can
+/// be placed earlier if the minimum VGPR budget can afford it).
+///
//===----------------------------------------------------------------------===//
#include "AMDGPUWMMASchedule.h"
@@ -20,7 +31,6 @@
#include "SIInstrInfo.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/ScheduleDAGInstrs.h"
-#include "llvm/Support/Debug.h"
#include <optional>
#define DEBUG_TYPE "amdgpu-wmma-sched"
@@ -28,12 +38,24 @@ using namespace llvm;
namespace {
-// A ds_load plus the program order positions (among WMMAs) of its
-// earliest and latest WMMA consumer.
+// A single ds_load and their order among the WMMAs.
struct LoadInfo {
SUnit *SU;
- unsigned MinPos = UINT_MAX; // earliest consumer (UINT_MAX means none in this region)
- unsigned MaxPos = 0; // latest consumer
+ unsigned MinPos = UINT_MAX; // earliest WMMA consumer (UINT_MAX means none in region)
+ unsigned MaxPos = 0; // latest WMMA consumer
+ long LatestCycle = 0; // as late as possible cycle
+};
+
+// A fragment: the wide vreg several ds_loads build (for example - a vreg_512
+// from four DS_READ_B128). This is the unit for VGPR pressure - the DS_READ
+// subloads share one register, so counting per ds_load instead of by fragments
+// would multiply the pressure.
+struct FragInfo {
+ unsigned VGPRs = 0;
+ unsigned MinPos = UINT_MAX;
+ unsigned MaxPos = 0;
+ long LatestCycle = LONG_MAX; // earliest subload's as late as possible cycle
+ SmallVector<SUnit *, 4> Subloads;
};
class WMMASchedule : public ScheduleDAGMutation {
@@ -56,9 +78,9 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
const SIInstrInfo *TII = ST.getInstrInfo();
// Gather WMMAs (numbered in program order) and ds_loads.
- MapVector<SUnit *, unsigned> Wmmas; // WMMA SUnit and its program position relative to one another
+ MapVector<SUnit *, unsigned> Wmmas; // Ordered WMMA SUnits
SmallVector<LoadInfo> Loads;
- std::optional<unsigned> LoadLatency = 64;
+ std::optional<unsigned> LoadLatency;
std::optional<unsigned> WmmaLatency;
std::optional<double> LDSBandwidth;
@@ -93,16 +115,14 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
auto [PrevSU, _] = *Wmmas.begin();
for (auto *It = std::next(Wmmas.begin()); It != Wmmas.end(); ++It) {
auto [SU, _] = *It;
- bool Success = DAG->addEdge(SU, SDep(PrevSU, SDep::Artificial));
- LLVM_DEBUG(dbgs() << "wmma SU" << SU->NodeNum << " <- after wmma SU"
- << PrevSU->NodeNum << (Success ? "\n" : " FAIL (cycle)\n"));
+ DAG->addEdge(SU, SDep(PrevSU, SDep::Artificial));
PrevSU = SU;
}
- // For each load, find the earliest and latest consuming WMMA positions.
- // Additionally correct the latency of the ds_load -> earliest WMMA consumer
- // data edge.
- for (LoadInfo& LI : Loads) {
+ // For each load, find earliest and latest consuming WMMA positions, and
+ // correct the ds_load -> earliest consumer data edge latency (Both the
+ // Succs and Preds SDep is updated)
+ for (LoadInfo &LI : Loads) {
for (const SDep &D : LI.SU->Succs) {
if (D.getKind() != SDep::Data)
continue;
@@ -131,7 +151,9 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
llvm::stable_sort(Loads, [](const LoadInfo &A, const LoadInfo &B) {
return A.MinPos < B.MinPos;
});
- SUnit* Prev = nullptr;
+
+ // Chain consecutive loads with an LDS bandwidth latency
+ SUnit *Prev = nullptr;
for (LoadInfo &LI : Loads) {
if (LI.MinPos == UINT_MAX)
continue;
@@ -143,47 +165,81 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
Prev = LI.SU;
}
- // MaxPos in order
- std::vector<bool> Present(Wmmas.size(), false);
- for (const LoadInfo &LI : Loads)
- // Check if there's a consumer for this load in the region
+ // Determing each load's as late as possible cycle - this means the
+ // latest cycle that still meets the load latency, then pushed earlier
+ // if the ds_load -> ds_load edges requires spacing (ds_loads cannot be
+ // too close to each other or it could overwhelm the LDS bus and lead to
+ // the program being memory bound).
+ for (LoadInfo &LI : Loads)
if (LI.MinPos != UINT_MAX)
- Present[LI.MaxPos] = true;
-
- // Latest position that's <= position Pos at which some load's register frees
- // A load's register becomes free at its MaxPos, which is its last WMMA consumer.
- std::vector<std::optional<unsigned>> DeadBy(Wmmas.size(), std::nullopt);
- std::optional<unsigned> Latest;
- for (unsigned Pos = 0; Pos < Wmmas.size(); ++Pos) {
- if (Present[Pos])
- Latest = Pos;
- DeadBy[Pos] = Latest;
+ LI.LatestCycle = (long)LI.MinPos * (*WmmaLatency) - (long)(*LoadLatency);
+ long PrevLatest = LONG_MAX;
+ for (int I = (int)Loads.size() - 1; I >= 0; --I) {
+ LoadInfo &LI = Loads[I];
+ if (LI.MinPos == UINT_MAX)
+ continue;
+ long Spaced = PrevLatest - (long)(*LDSBandwidth);
+ if (Spaced < LI.LatestCycle)
+ LI.LatestCycle = Spaced;
+ PrevLatest = LI.LatestCycle;
}
- // Create the edges that constrain where the ds_loads can be placed
- // minimum distance of a load is LI.MinPos - Dist
- // maximum distance of a load is DeadBy[LI.MinPos - Dist]
- unsigned Dist = std::max(1u, static_cast<unsigned>(std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency)));
- for (const LoadInfo &LI : Loads) {
- if (LI.MinPos == UINT_MAX || LI.MinPos < Dist)
+ // Group subloads into fragments and build the live range histogram
+ // with a schedule as late as possible. Each fragment is live from
+ // its earliest subload to its last WMMA consumer. The peak of the
+ // histogram is the minimum VGPRs needed.
+ MapVector<Register, FragInfo> Frags;
+ for (LoadInfo &LI : Loads) {
+ if (LI.MinPos == UINT_MAX)
continue;
-
- // Minimum distance edge
- // Load scheduled before this point (heuristically)
- long MinDist = LI.MinPos - Dist;
-
- // Maximum distance edge
- // Load scheduled after this point
- bool MaxSuccess = true;
- std::optional<unsigned> MaxDist = DeadBy[MinDist];
- if (MaxDist && *MaxDist < MinDist)
- MaxSuccess = DAG->addEdge(LI.SU, SDep(Wmmas.begin()[*MaxDist].first, SDep::Artificial));
-
- LLVM_DEBUG(dbgs() << "load SU" << LI.SU->NodeNum << " Win=[" << (MaxDist ? *MaxDist : -1) << ","
- << MinDist << "] MinPos=" << LI.MinPos << " MaxPos=" << LI.MaxPos
- << (MaxSuccess ? "\n" : " (edge REJECTED: cycle)\n"));
+ Register R = LI.SU->getInstr()->getOperand(0).getReg();
+ FragInfo &F = Frags[R];
+ if (F.Subloads.empty() && R.isVirtual())
+ F.VGPRs = TRI.getRegSizeInBits(*MRI.getRegClass(R)) / 32;
+ F.MinPos = std::min(F.MinPos, LI.MinPos);
+ F.MaxPos = std::max(F.MaxPos, LI.MaxPos);
+ F.LatestCycle = std::min(F.LatestCycle, LI.LatestCycle);
+ F.Subloads.push_back(LI.SU);
+ }
+
+ std::vector<unsigned> Hist(Wmmas.size(), 0);
+ for (auto &KV : Frags) {
+ FragInfo &F = KV.second;
+ long Pos = F.LatestCycle / (long)(*WmmaLatency);
+ unsigned StartPos = Pos < 0 ? 0 : (unsigned)Pos;
+ for (unsigned P = StartPos; P <= F.MaxPos && P < Wmmas.size(); ++P)
+ Hist[P] += F.VGPRs;
}
+ unsigned Budget = 0;
+ for (unsigned P = 0; P < Wmmas.size(); ++P)
+ Budget = std::max(Budget, Hist[P]);
+
+ // For each fragment (in order), find the earliest position it
+ // can be placed so the live set never exceeds the budget, then
+ // add a WMMAS[Earliest] -> ds_load edge - this is what leads to
+ // the debunching.
+ for (auto &KV : Frags) {
+ FragInfo &F = KV.second;
+ long Pos = F.LatestCycle / (long)(*WmmaLatency);
+ unsigned LateStartPos = Pos < 0 ? 0 : (unsigned)Pos;
+ unsigned Earliest = LateStartPos;
+ for (int P = (int)LateStartPos - 1; P >= 0; --P) {
+ if (Hist[(unsigned)P] + F.VGPRs <= Budget)
+ Earliest = (unsigned)P;
+ else
+ break;
+ }
+ // Update the histogram so later fragments don't schedule earlier and
+ // exceed the budget.
+ for (unsigned P = Earliest; P < LateStartPos; ++P)
+ Hist[P] += F.VGPRs;
+ // No need to add an edge if the load can be scheduled at the beginning.
+ if (Earliest == 0)
+ continue;
+ for (SUnit *L : F.Subloads)
+ DAG->addEdge(L, SDep(Wmmas.begin()[Earliest].first, SDep::Artificial));
+ }
}
} // end namespace
>From 0a694f3727ef5f1460e132bac7d794834a40fb3f Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Mon, 15 Jun 2026 07:42:01 +0000
Subject: [PATCH 9/9] clang format
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 209 +++++++++---------
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 23 +-
2 files changed, 112 insertions(+), 120 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index edbc8cafbde54..0cfca996e874e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -184,13 +184,13 @@ class AMDGPUCodeGenPassBuilder
class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
public:
SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
- : RegisterRegAllocBase(N, D, C) {}
+ : RegisterRegAllocBase(N, D, C) {}
};
class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
public:
VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
- : RegisterRegAllocBase(N, D, C) {}
+ : RegisterRegAllocBase(N, D, C) {}
};
class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
@@ -233,19 +233,21 @@ static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
static SGPRRegisterRegAlloc
-defaultSGPRRegAlloc("default",
- "pick SGPR register allocator based on -O option",
- useDefaultRegisterAllocator);
+ defaultSGPRRegAlloc("default",
+ "pick SGPR register allocator based on -O option",
+ useDefaultRegisterAllocator);
static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
RegisterPassParser<SGPRRegisterRegAlloc>>
-SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
- cl::desc("Register allocator to use for SGPRs"));
+ SGPRRegAlloc("sgpr-regalloc", cl::Hidden,
+ cl::init(&useDefaultRegisterAllocator),
+ cl::desc("Register allocator to use for SGPRs"));
static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
RegisterPassParser<VGPRRegisterRegAlloc>>
-VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
- cl::desc("Register allocator to use for VGPRs"));
+ VGPRRegAlloc("vgpr-regalloc", cl::Hidden,
+ cl::init(&useDefaultRegisterAllocator),
+ cl::desc("Register allocator to use for VGPRs"));
static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
RegisterPassParser<WWMRegisterRegAlloc>>
@@ -373,22 +375,25 @@ static FunctionPass *createFastWWMRegisterAllocator() {
return createFastRegisterAllocator(onlyAllocateWWMRegs, false);
}
-static SGPRRegisterRegAlloc basicRegAllocSGPR(
- "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
-static SGPRRegisterRegAlloc greedyRegAllocSGPR(
- "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
-
-static SGPRRegisterRegAlloc fastRegAllocSGPR(
- "fast", "fast register allocator", createFastSGPRRegisterAllocator);
+static SGPRRegisterRegAlloc basicRegAllocSGPR("basic",
+ "basic register allocator",
+ createBasicSGPRRegisterAllocator);
+static SGPRRegisterRegAlloc
+ greedyRegAllocSGPR("greedy", "greedy register allocator",
+ createGreedySGPRRegisterAllocator);
+static SGPRRegisterRegAlloc fastRegAllocSGPR("fast", "fast register allocator",
+ createFastSGPRRegisterAllocator);
-static VGPRRegisterRegAlloc basicRegAllocVGPR(
- "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
-static VGPRRegisterRegAlloc greedyRegAllocVGPR(
- "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
+static VGPRRegisterRegAlloc basicRegAllocVGPR("basic",
+ "basic register allocator",
+ createBasicVGPRRegisterAllocator);
+static VGPRRegisterRegAlloc
+ greedyRegAllocVGPR("greedy", "greedy register allocator",
+ createGreedyVGPRRegisterAllocator);
-static VGPRRegisterRegAlloc fastRegAllocVGPR(
- "fast", "fast register allocator", createFastVGPRRegisterAllocator);
+static VGPRRegisterRegAlloc fastRegAllocVGPR("fast", "fast register allocator",
+ createFastVGPRRegisterAllocator);
static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
"basic register allocator",
createBasicWWMRegisterAllocator);
@@ -405,14 +410,14 @@ static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
} // anonymous namespace
static cl::opt<bool>
-EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
- cl::desc("Run early if-conversion"),
- cl::init(false));
+ EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
+ cl::desc("Run early if-conversion"),
+ cl::init(false));
static cl::opt<bool>
-OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
- cl::desc("Run pre-RA exec mask optimizations"),
- cl::init(true));
+ OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
+ cl::desc("Run pre-RA exec mask optimizations"),
+ cl::init(true));
static cl::opt<bool>
LowerCtorDtor("amdgpu-lower-global-ctor-dtor",
@@ -420,32 +425,27 @@ static cl::opt<bool>
cl::init(true), cl::Hidden);
// Option to disable vectorizer for tests.
-static cl::opt<bool> EnableLoadStoreVectorizer(
- "amdgpu-load-store-vectorizer",
- cl::desc("Enable load store vectorizer"),
- cl::init(true),
- cl::Hidden);
+static cl::opt<bool>
+ EnableLoadStoreVectorizer("amdgpu-load-store-vectorizer",
+ cl::desc("Enable load store vectorizer"),
+ cl::init(true), cl::Hidden);
// Option to control global loads scalarization
-static cl::opt<bool> ScalarizeGlobal(
- "amdgpu-scalarize-global-loads",
- cl::desc("Enable global load scalarization"),
- cl::init(true),
- cl::Hidden);
+static cl::opt<bool>
+ ScalarizeGlobal("amdgpu-scalarize-global-loads",
+ cl::desc("Enable global load scalarization"),
+ cl::init(true), cl::Hidden);
// Option to run internalize pass.
static cl::opt<bool> InternalizeSymbols(
- "amdgpu-internalize-symbols",
- cl::desc("Enable elimination of non-kernel functions and unused globals"),
- cl::init(false),
- cl::Hidden);
+ "amdgpu-internalize-symbols",
+ cl::desc("Enable elimination of non-kernel functions and unused globals"),
+ cl::init(false), cl::Hidden);
// Option to inline all early.
-static cl::opt<bool> EarlyInlineAll(
- "amdgpu-early-inline-all",
- cl::desc("Inline all functions early"),
- cl::init(false),
- cl::Hidden);
+static cl::opt<bool> EarlyInlineAll("amdgpu-early-inline-all",
+ cl::desc("Inline all functions early"),
+ cl::init(false), cl::Hidden);
static cl::opt<bool> RemoveIncompatibleFunctions(
"amdgpu-enable-remove-incompatible-functions", cl::Hidden,
@@ -453,39 +453,35 @@ static cl::opt<bool> RemoveIncompatibleFunctions(
"use features not supported by the target GPU"),
cl::init(true));
-static cl::opt<bool> EnableSDWAPeephole(
- "amdgpu-sdwa-peephole",
- cl::desc("Enable SDWA peepholer"),
- cl::init(true));
+static cl::opt<bool> EnableSDWAPeephole("amdgpu-sdwa-peephole",
+ cl::desc("Enable SDWA peepholer"),
+ cl::init(true));
-static cl::opt<bool> EnableDPPCombine(
- "amdgpu-dpp-combine",
- cl::desc("Enable DPP combiner"),
- cl::init(true));
+static cl::opt<bool> EnableDPPCombine("amdgpu-dpp-combine",
+ cl::desc("Enable DPP combiner"),
+ cl::init(true));
// Enable address space based alias analysis
-static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
- cl::desc("Enable AMDGPU Alias Analysis"),
- cl::init(true));
+static cl::opt<bool>
+ EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
+ cl::desc("Enable AMDGPU Alias Analysis"),
+ cl::init(true));
// Enable lib calls simplifications
-static cl::opt<bool> EnableLibCallSimplify(
- "amdgpu-simplify-libcall",
- cl::desc("Enable amdgpu library simplifications"),
- cl::init(true),
- cl::Hidden);
+static cl::opt<bool>
+ EnableLibCallSimplify("amdgpu-simplify-libcall",
+ cl::desc("Enable amdgpu library simplifications"),
+ cl::init(true), cl::Hidden);
static cl::opt<bool> EnableLowerKernelArguments(
- "amdgpu-ir-lower-kernel-arguments",
- cl::desc("Lower kernel argument loads in IR pass"),
- cl::init(true),
- cl::Hidden);
+ "amdgpu-ir-lower-kernel-arguments",
+ cl::desc("Lower kernel argument loads in IR pass"), cl::init(true),
+ cl::Hidden);
static cl::opt<bool> EnableRegReassign(
- "amdgpu-reassign-regs",
- cl::desc("Enable register reassign optimizations on gfx10+"),
- cl::init(true),
- cl::Hidden);
+ "amdgpu-reassign-regs",
+ cl::desc("Enable register reassign optimizations on gfx10+"),
+ cl::init(true), cl::Hidden);
static cl::opt<bool> OptVGPRLiveRange(
"amdgpu-opt-vgpr-liverange",
@@ -503,11 +499,10 @@ static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(
clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));
// Enable Mode register optimization
-static cl::opt<bool> EnableSIModeRegisterPass(
- "amdgpu-mode-register",
- cl::desc("Enable mode register pass"),
- cl::init(true),
- cl::Hidden);
+static cl::opt<bool>
+ EnableSIModeRegisterPass("amdgpu-mode-register",
+ cl::desc("Enable mode register pass"),
+ cl::init(true), cl::Hidden);
// Enable GFX11+ s_delay_alu insertion
static cl::opt<bool>
@@ -523,19 +518,16 @@ static cl::opt<bool>
// Option is used in lit tests to prevent deadcoding of patterns inspected.
static cl::opt<bool>
-EnableDCEInRA("amdgpu-dce-in-ra",
- cl::init(true), cl::Hidden,
- cl::desc("Enable machine DCE inside regalloc"));
+ EnableDCEInRA("amdgpu-dce-in-ra", cl::init(true), cl::Hidden,
+ cl::desc("Enable machine DCE inside regalloc"));
static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",
cl::desc("Adjust wave priority"),
cl::init(false), cl::Hidden);
-static cl::opt<bool> EnableScalarIRPasses(
- "amdgpu-scalar-ir-passes",
- cl::desc("Enable scalar IR passes"),
- cl::init(true),
- cl::Hidden);
+static cl::opt<bool> EnableScalarIRPasses("amdgpu-scalar-ir-passes",
+ cl::desc("Enable scalar IR passes"),
+ cl::init(true), cl::Hidden);
static cl::opt<bool> EnableLowerExecSync(
"amdgpu-enable-lower-exec-sync",
@@ -559,10 +551,10 @@ static cl::opt<bool, true> EnableLowerModuleLDS(
cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true),
cl::Hidden);
-static cl::opt<bool> EnablePreRAOptimizations(
- "amdgpu-enable-pre-ra-optimizations",
- cl::desc("Enable Pre-RA optimizations pass"), cl::init(true),
- cl::Hidden);
+static cl::opt<bool>
+ EnablePreRAOptimizations("amdgpu-enable-pre-ra-optimizations",
+ cl::desc("Enable Pre-RA optimizations pass"),
+ cl::init(true), cl::Hidden);
static cl::opt<bool> EnablePromoteKernelArguments(
"amdgpu-enable-promote-kernel-arguments",
@@ -621,10 +613,10 @@ static cl::opt<bool> EnableRewritePartialRegUses(
cl::desc("Enable rewrite partial reg uses pass"), cl::init(true),
cl::Hidden);
-static cl::opt<bool> EnableHipStdPar(
- "amdgpu-enable-hipstdpar",
- cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false),
- cl::Hidden);
+static cl::opt<bool>
+ EnableHipStdPar("amdgpu-enable-hipstdpar",
+ cl::desc("Enable HIP Standard Parallelism Offload support"),
+ cl::init(false), cl::Hidden);
static cl::opt<bool>
EnableAMDGPUAttributor("amdgpu-attributor-enable",
@@ -750,8 +742,8 @@ static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
static ScheduleDAGInstrs *
createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
- ScheduleDAGMILive *DAG =
- new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));
+ ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
+ C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));
DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.shouldClusterStores())
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
@@ -817,14 +809,13 @@ createIterativeILPMachineScheduler(MachineSchedContext *C) {
return DAG;
}
-static MachineSchedRegistry
-SISchedRegistry("si", "Run SI's custom scheduler",
- createSIMachineScheduler);
+static MachineSchedRegistry SISchedRegistry("si", "Run SI's custom scheduler",
+ createSIMachineScheduler);
static MachineSchedRegistry
-GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
- "Run GCN scheduler to maximize occupancy",
- createGCNMaxOccupancyMachineScheduler);
+ GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
+ "Run GCN scheduler to maximize occupancy",
+ createGCNMaxOccupancyMachineScheduler);
static MachineSchedRegistry
GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
@@ -1497,7 +1488,7 @@ void AMDGPUPassConfig::addIRPasses() {
AAResults &AAR) {
if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
AAR.addAAResult(WrapperPass->getResult());
- }));
+ }));
}
if (TM.getTargetTriple().isAMDGCN()) {
@@ -2111,8 +2102,8 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
AMDGPU::SGPR_32RegClass,
MFI->ArgInfo.PrivateSegmentSize, 0, 0) ||
parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
- AMDGPU::SGPR_32RegClass,
- MFI->ArgInfo.LDSKernelId, 0, 1) ||
+ AMDGPU::SGPR_32RegClass, MFI->ArgInfo.LDSKernelId,
+ 0, 1) ||
parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
0, 1) ||
@@ -2135,14 +2126,14 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
AMDGPU::SReg_64RegClass,
MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
- AMDGPU::VGPR_32RegClass,
- MFI->ArgInfo.WorkItemIDX, 0, 0) ||
+ AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDX,
+ 0, 0) ||
parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
- AMDGPU::VGPR_32RegClass,
- MFI->ArgInfo.WorkItemIDY, 0, 0) ||
+ AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDY,
+ 0, 0) ||
parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
- AMDGPU::VGPR_32RegClass,
- MFI->ArgInfo.WorkItemIDZ, 0, 0)))
+ AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDZ,
+ 0, 0)))
return true;
// Parse FirstKernArgPreloadReg separately, since it's a Register,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index 85ea68f4ebf62..4cdd0f5d1ecca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -20,7 +20,7 @@
/// - Add WMMA -> ds_load edges to stop loads from being bunched at
/// the start of the block
/// - Build a live range histogram of the A/B operand fragments under
-/// an as late as possible schedule, recording the minimum VGPRs
+/// an as late as possible schedule, recording the minimum VGPRs
/// needed for such a schedule (so the WMMA -> ds_load edges can
/// be placed earlier if the minimum VGPR budget can afford it).
///
@@ -41,9 +41,10 @@ namespace {
// A single ds_load and their order among the WMMAs.
struct LoadInfo {
SUnit *SU;
- unsigned MinPos = UINT_MAX; // earliest WMMA consumer (UINT_MAX means none in region)
- unsigned MaxPos = 0; // latest WMMA consumer
- long LatestCycle = 0; // as late as possible cycle
+ unsigned MinPos =
+ UINT_MAX; // earliest WMMA consumer (UINT_MAX means none in region)
+ unsigned MaxPos = 0; // latest WMMA consumer
+ long LatestCycle = 0; // as late as possible cycle
};
// A fragment: the wide vreg several ds_loads build (for example - a vreg_512
@@ -54,7 +55,7 @@ struct FragInfo {
unsigned VGPRs = 0;
unsigned MinPos = UINT_MAX;
unsigned MaxPos = 0;
- long LatestCycle = LONG_MAX; // earliest subload's as late as possible cycle
+ long LatestCycle = LONG_MAX; // earliest subload's as late as possible cycle
SmallVector<SUnit *, 4> Subloads;
};
@@ -99,7 +100,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather DS_LOADs
if (TII->isDS(*MI) && MI->mayLoad()) {
- if (!LoadLatency)
+ if (!LoadLatency)
LoadLatency = SM->computeInstrLatency(MI);
if (!LDSBandwidth)
LDSBandwidth = std::ceil(SM->computeReciprocalThroughput(MI));
@@ -120,7 +121,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
}
// For each load, find earliest and latest consuming WMMA positions, and
- // correct the ds_load -> earliest consumer data edge latency (Both the
+ // correct the ds_load -> earliest consumer data edge latency (Both the
// Succs and Preds SDep is updated)
for (LoadInfo &LI : Loads) {
for (const SDep &D : LI.SU->Succs) {
@@ -133,7 +134,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
LI.MinPos = std::min(LI.MinPos, It->second);
LI.MaxPos = std::max(LI.MaxPos, It->second);
}
- if (LI.MinPos == UINT_MAX)
+ if (LI.MinPos == UINT_MAX)
continue;
SUnit *EarliestConsumer = Wmmas.begin()[LI.MinPos].first;
// Correct latency of edges between ds_load and earliest WMMA consumer
@@ -155,7 +156,7 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Chain consecutive loads with an LDS bandwidth latency
SUnit *Prev = nullptr;
for (LoadInfo &LI : Loads) {
- if (LI.MinPos == UINT_MAX)
+ if (LI.MinPos == UINT_MAX)
continue;
if (Prev) {
SDep D(Prev, SDep::Artificial);
@@ -185,8 +186,8 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
}
// Group subloads into fragments and build the live range histogram
- // with a schedule as late as possible. Each fragment is live from
- // its earliest subload to its last WMMA consumer. The peak of the
+ // with a schedule as late as possible. Each fragment is live from
+ // its earliest subload to its last WMMA consumer. The peak of the
// histogram is the minimum VGPRs needed.
MapVector<Register, FragInfo> Frags;
for (LoadInfo &LI : Loads) {
More information about the llvm-commits
mailing list