[llvm] [AMDGPU] DAG Mutation to solve load bunching in outer product matrix multiplications (PR #203095)
Axel Sorenson via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 12:46:38 PDT 2026
https://github.com/axelcool1234 updated https://github.com/llvm/llvm-project/pull/203095
>From a26e9913e052b09a5e9712a853b9d40bc22e9802 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 4 Jun 2026 23:36:28 +0000
Subject: [PATCH 1/4] DAG Mutation to solve ds_load bunching in outer product
matmuls
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 119 ++++++++++++++++++
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h | 24 ++++
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
4 files changed, 146 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 65317016c6390..c4c4977218746 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -39,6 +39,7 @@
#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUUnifyDivergentExitNodes.h"
#include "AMDGPUWaitSGPRHazards.h"
+#include "AMDGPUWMMASchedule.h"
#include "GCNDPPCombine.h"
#include "GCNIterativeScheduler.h"
#include "GCNNSAReassign.h"
@@ -759,6 +760,7 @@ createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));
DAG->addMutation(createAMDGPUHazardLatencyDAGMutation(C->MF));
+ DAG->addMutation(createAMDGPUWMMAScheduleDAGMutation(C->MF));
return DAG;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
new file mode 100644
index 0000000000000..e367ad9a22659
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -0,0 +1,119 @@
+//===--- AMDGPUWMMASchedule.cpp - AMDGPU WMMA Schedule Adjustment ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This file contains a DAG scheduling mutation to add additional
+/// edges between ds_load instructions and wmma instructions that
+/// occur a certain amount away from the actual wmma consumer of
+/// said ds_load. This forces the ds_load to properly prefetch
+/// and prevent early bunching of ds_loads that then lead to long
+/// stalls.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUWMMASchedule.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/Support/Debug.h"
+#include <optional>
+#define DEBUG_TYPE "amdgpu-wmma-sched"
+
+using namespace llvm;
+
+namespace {
+
+class WMMASchedule : public ScheduleDAGMutation {
+private:
+ const GCNSubtarget &ST;
+ const SIRegisterInfo &TRI;
+ const MachineRegisterInfo &MRI;
+
+public:
+ WMMASchedule(MachineFunction *MF)
+ : ST(MF->getSubtarget<GCNSubtarget>()), TRI(*ST.getRegisterInfo()),
+ MRI(MF->getRegInfo()) {}
+ void apply(ScheduleDAGInstrs *DAG) override;
+};
+
+void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
+ if (!ST.hasGFX1250Insts()) return;
+ const TargetSchedModel *SM = DAG->getSchedModel();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size() << "SUnits\n");
+
+ SmallVector<SUnit*> Loads;
+ MapVector<SUnit*, unsigned> Wmmas;
+
+ std::optional<unsigned> LoadLatency = std::nullopt;
+ std::optional<unsigned> WmmaLatency = std::nullopt;
+
+ // Gather all WMMAs and DS_LOADs
+ for(auto &SU : DAG->SUnits) {
+ MachineInstr* MI = SU.getInstr();
+ if(!MI) continue;
+
+ // Gather WMMAs
+ if(TII->isMFMAorWMMA(*MI)) {
+ if(WmmaLatency == std::nullopt) WmmaLatency = SM->computeInstrLatency(MI);
+ Wmmas.insert({&SU, Wmmas.size()});
+ continue;
+ }
+
+ // Gather DS_LOADs
+ if(TII->isDS(*MI) && MI->mayLoad()) {
+ if(LoadLatency == std::nullopt) LoadLatency = SM->computeInstrLatency(MI);
+ Loads.push_back(&SU);
+ }
+ }
+
+ // Calculate how many WMMAs away from consuming WMMA a load must be
+ // before it will certainly ready for consumer
+ unsigned Dist;
+ if(LoadLatency && WmmaLatency) {
+ Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1) Dist = 1;
+ } else {
+ return; // Either missing Wmmas or Loads. No point continuing.
+ }
+ LLVM_DEBUG(dbgs() << "Dist " << Dist << "\n");
+
+ // For every load, determine earliest WMMA reliant on it,
+ // and add an anchor.
+ for(SUnit* L : Loads){
+ SUnit* Earliest = nullptr;
+ unsigned EarliestPos = UINT_MAX;
+ for(const SDep& D : L->Succs){
+ if(D.getKind() != SDep::Data) continue;
+ SUnit* S = D.getSUnit();
+ auto *It = Wmmas.find(S);
+ if(It == Wmmas.end()) continue;
+ if(It->second < EarliestPos) {
+ EarliestPos = It->second;
+ Earliest = S;
+ }
+ };
+ LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
+ << (Earliest ? (int)Earliest->NodeNum : -1)
+ << " (pos " << EarliestPos << ")\n");
+ if(Earliest && EarliestPos >= Dist) {
+ LLVM_DEBUG(dbgs() << "Window Created!\n");
+ SUnit* Anchor = Wmmas.begin()[EarliestPos - Dist].first;
+ bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
+ LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU" << Anchor->NodeNum
+ << (Ok ? "\n" : " (REJECTED: cycle)\n"));
+ };
+ };
+}
+
+} // end namespace
+
+std::unique_ptr<ScheduleDAGMutation>
+llvm::createAMDGPUWMMAScheduleDAGMutation(MachineFunction *MF) {
+ return std::make_unique<WMMASchedule>(MF);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
new file mode 100644
index 0000000000000..781f4b3b1ac86
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.h
@@ -0,0 +1,24 @@
+//===- AMDGPUWMMASchedule.h - WMMA Schedule Adjustment ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
+
+#include "llvm/CodeGen/ScheduleDAGMutation.h"
+#include <memory>
+
+namespace llvm {
+
+class MachineFunction;
+
+std::unique_ptr<ScheduleDAGMutation>
+createAMDGPUWMMAScheduleDAGMutation(MachineFunction *MF);
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUWMMASCHEDULE_H
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index ae8f1c0fad5ba..82f129b74076c 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -122,6 +122,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUTargetTransformInfo.cpp
AMDGPUWaitcntUtils.cpp
AMDGPUWaitSGPRHazards.cpp
+ AMDGPUWMMASchedule.cpp
AMDGPUUnifyDivergentExitNodes.cpp
R600MachineCFGStructurizer.cpp
GCNCreateVOPD.cpp
>From 7e16bae438d4ac052f5b57b4000b2a0579c05778 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Wed, 10 Jun 2026 20:23:37 +0000
Subject: [PATCH 2/4] Clang format
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +-
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 69 +++++++++++--------
2 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index c4c4977218746..edbc8cafbde54 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -38,8 +38,8 @@
#include "AMDGPUTargetObjectFile.h"
#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUUnifyDivergentExitNodes.h"
-#include "AMDGPUWaitSGPRHazards.h"
#include "AMDGPUWMMASchedule.h"
+#include "AMDGPUWaitSGPRHazards.h"
#include "GCNDPPCombine.h"
#include "GCNIterativeScheduler.h"
#include "GCNNSAReassign.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index e367ad9a22659..d87326ba00b49 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -42,42 +42,48 @@ class WMMASchedule : public ScheduleDAGMutation {
};
void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
- if (!ST.hasGFX1250Insts()) return;
+ if (!ST.hasGFX1250Insts())
+ return;
const TargetSchedModel *SM = DAG->getSchedModel();
const SIInstrInfo *TII = ST.getInstrInfo();
- LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size() << "SUnits\n");
+ LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size()
+ << "SUnits\n");
- SmallVector<SUnit*> Loads;
- MapVector<SUnit*, unsigned> Wmmas;
+ SmallVector<SUnit *> Loads;
+ MapVector<SUnit *, unsigned> Wmmas;
std::optional<unsigned> LoadLatency = std::nullopt;
std::optional<unsigned> WmmaLatency = std::nullopt;
-
+
// Gather all WMMAs and DS_LOADs
- for(auto &SU : DAG->SUnits) {
- MachineInstr* MI = SU.getInstr();
- if(!MI) continue;
+ for (auto &SU : DAG->SUnits) {
+ MachineInstr *MI = SU.getInstr();
+ if (!MI)
+ continue;
// Gather WMMAs
- if(TII->isMFMAorWMMA(*MI)) {
- if(WmmaLatency == std::nullopt) WmmaLatency = SM->computeInstrLatency(MI);
+ if (TII->isMFMAorWMMA(*MI)) {
+ if (WmmaLatency == std::nullopt)
+ WmmaLatency = SM->computeInstrLatency(MI);
Wmmas.insert({&SU, Wmmas.size()});
continue;
}
// Gather DS_LOADs
- if(TII->isDS(*MI) && MI->mayLoad()) {
- if(LoadLatency == std::nullopt) LoadLatency = SM->computeInstrLatency(MI);
+ if (TII->isDS(*MI) && MI->mayLoad()) {
+ if (LoadLatency == std::nullopt)
+ LoadLatency = SM->computeInstrLatency(MI);
Loads.push_back(&SU);
}
}
- // Calculate how many WMMAs away from consuming WMMA a load must be
+ // Calculate how many WMMAs away from consuming WMMA a load must be
// before it will certainly ready for consumer
unsigned Dist;
- if(LoadLatency && WmmaLatency) {
- Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
- if (Dist < 1) Dist = 1;
+ if (LoadLatency && WmmaLatency) {
+ Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1)
+ Dist = 1;
} else {
return; // Either missing Wmmas or Loads. No point continuing.
}
@@ -85,28 +91,31 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// For every load, determine earliest WMMA reliant on it,
// and add an anchor.
- for(SUnit* L : Loads){
- SUnit* Earliest = nullptr;
+ for (SUnit *L : Loads) {
+ SUnit *Earliest = nullptr;
unsigned EarliestPos = UINT_MAX;
- for(const SDep& D : L->Succs){
- if(D.getKind() != SDep::Data) continue;
- SUnit* S = D.getSUnit();
+ for (const SDep &D : L->Succs) {
+ if (D.getKind() != SDep::Data)
+ continue;
+ SUnit *S = D.getSUnit();
auto *It = Wmmas.find(S);
- if(It == Wmmas.end()) continue;
- if(It->second < EarliestPos) {
- EarliestPos = It->second;
+ if (It == Wmmas.end())
+ continue;
+ if (It->second < EarliestPos) {
+ EarliestPos = It->second;
Earliest = S;
}
};
LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
- << (Earliest ? (int)Earliest->NodeNum : -1)
- << " (pos " << EarliestPos << ")\n");
- if(Earliest && EarliestPos >= Dist) {
+ << (Earliest ? (int)Earliest->NodeNum : -1) << " (pos "
+ << EarliestPos << ")\n");
+ if (Earliest && EarliestPos >= Dist) {
LLVM_DEBUG(dbgs() << "Window Created!\n");
- SUnit* Anchor = Wmmas.begin()[EarliestPos - Dist].first;
+ SUnit *Anchor = Wmmas.begin()[EarliestPos - Dist].first;
bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
- LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU" << Anchor->NodeNum
- << (Ok ? "\n" : " (REJECTED: cycle)\n"));
+ LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU"
+ << Anchor->NodeNum
+ << (Ok ? "\n" : " (REJECTED: cycle)\n"));
};
};
}
>From d0c4dade1e4280a67bd43f84286709c4ef525846 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Wed, 10 Jun 2026 23:25:51 +0000
Subject: [PATCH 3/4] Maximum and minimum distance
---
llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp | 125 +++++++++++-------
1 file changed, 76 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
index d87326ba00b49..fba0109a2fcca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWMMASchedule.cpp
@@ -28,6 +28,14 @@ using namespace llvm;
namespace {
+// A ds_load plus the program order positions (among WMMAs) of its
+// earliest and latest WMMA consumer.
+struct LoadInfo {
+ SUnit *SU;
+ unsigned MinPos = UINT_MAX; // earliest consumer (UINT_MAX means none in this region)
+ unsigned MaxPos = 0; // latest consumer
+};
+
class WMMASchedule : public ScheduleDAGMutation {
private:
const GCNSubtarget &ST;
@@ -46,24 +54,20 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
return;
const TargetSchedModel *SM = DAG->getSchedModel();
const SIInstrInfo *TII = ST.getInstrInfo();
- LLVM_DEBUG(dbgs() << "WMMASchedule running, " << DAG->SUnits.size()
- << "SUnits\n");
-
- SmallVector<SUnit *> Loads;
- MapVector<SUnit *, unsigned> Wmmas;
- std::optional<unsigned> LoadLatency = std::nullopt;
- std::optional<unsigned> WmmaLatency = std::nullopt;
+ // Gather WMMAs (numbered in program order) and ds_loads.
+ MapVector<SUnit *, unsigned> Wmmas; // WMMA SUnit and its program position relative to one another
+ SmallVector<LoadInfo> Loads;
+ std::optional<unsigned> LoadLatency, WmmaLatency;
- // Gather all WMMAs and DS_LOADs
- for (auto &SU : DAG->SUnits) {
+ for (SUnit &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
if (!MI)
continue;
// Gather WMMAs
if (TII->isMFMAorWMMA(*MI)) {
- if (WmmaLatency == std::nullopt)
+ if (!WmmaLatency)
WmmaLatency = SM->computeInstrLatency(MI);
Wmmas.insert({&SU, Wmmas.size()});
continue;
@@ -71,53 +75,76 @@ void WMMASchedule::apply(ScheduleDAGInstrs *DAG) {
// Gather DS_LOADs
if (TII->isDS(*MI) && MI->mayLoad()) {
- if (LoadLatency == std::nullopt)
+ if (!LoadLatency)
LoadLatency = SM->computeInstrLatency(MI);
- Loads.push_back(&SU);
+ Loads.push_back({&SU});
}
}
- // Calculate how many WMMAs away from consuming WMMA a load must be
- // before it will certainly ready for consumer
- unsigned Dist;
- if (LoadLatency && WmmaLatency) {
- Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
- if (Dist < 1)
- Dist = 1;
- } else {
- return; // Either missing Wmmas or Loads. No point continuing.
- }
- LLVM_DEBUG(dbgs() << "Dist " << Dist << "\n");
-
- // For every load, determine earliest WMMA reliant on it,
- // and add an anchor.
- for (SUnit *L : Loads) {
- SUnit *Earliest = nullptr;
- unsigned EarliestPos = UINT_MAX;
- for (const SDep &D : L->Succs) {
+ // The following means the DAG Mutation cannot do anything useful.
+ if (!LoadLatency || !WmmaLatency || Wmmas.empty())
+ return;
+
+ // The number of WMMAs that elapse during one load's latency
+ unsigned Dist = std::ceil(static_cast<double>(*LoadLatency) / *WmmaLatency);
+ if (Dist < 1)
+ Dist = 1;
+
+ // For each load, find the earliest and latest consuming WMMA positions.
+ for (LoadInfo& LI : Loads) {
+ for (const SDep &D : LI.SU->Succs) {
if (D.getKind() != SDep::Data)
continue;
- SUnit *S = D.getSUnit();
- auto *It = Wmmas.find(S);
+ auto *It = Wmmas.find(D.getSUnit());
+ // Check to see if successor is a WMMA
if (It == Wmmas.end())
continue;
- if (It->second < EarliestPos) {
- EarliestPos = It->second;
- Earliest = S;
- }
- };
- LLVM_DEBUG(dbgs() << "load SU" << L->NodeNum << " -> earliest WMMA SU"
- << (Earliest ? (int)Earliest->NodeNum : -1) << " (pos "
- << EarliestPos << ")\n");
- if (Earliest && EarliestPos >= Dist) {
- LLVM_DEBUG(dbgs() << "Window Created!\n");
- SUnit *Anchor = Wmmas.begin()[EarliestPos - Dist].first;
- bool Ok = DAG->addEdge(L, SDep(Anchor, SDep::Artificial));
- LLVM_DEBUG(dbgs() << " leash SU" << L->NodeNum << " after WMMA SU"
- << Anchor->NodeNum
- << (Ok ? "\n" : " (REJECTED: cycle)\n"));
- };
- };
+ LI.MinPos = std::min(LI.MinPos, It->second);
+ LI.MaxPos = std::max(LI.MaxPos, It->second);
+ }
+ }
+
+ // MaxPos in order
+ std::vector<bool> Present(Wmmas.size(), false);
+ for (const LoadInfo &LI : Loads)
+ // Check if there's a consumer for this load in the region
+ if (LI.MinPos != UINT_MAX)
+ Present[LI.MaxPos] = true;
+
+ // Latest position that's <= position Pos at which some load's register frees
+ // A load's register becomes free at its MaxPos, which is its last WMMA consumer.
+ std::vector<std::optional<unsigned>> DeadBy(Wmmas.size(), std::nullopt);
+ std::optional<unsigned> Latest;
+ for (unsigned Pos = 0; Pos < Wmmas.size(); ++Pos) {
+ if (Present[Pos])
+ Latest = Pos;
+ DeadBy[Pos] = Latest;
+ }
+
+ // Create the edges that constrain where the ds_loads can be placed
+ // minimum distance of a load is LI.MinPos - Dist
+ // maximum distance of a load is DeadBy[LI.MinPos - Dist]
+ for (const LoadInfo &LI : Loads) {
+ if (LI.MinPos == UINT_MAX || LI.MinPos < Dist)
+ continue;
+
+ // Minimum distance edge
+ // Load scheduled before this point
+ unsigned MinDist = LI.MinPos - Dist;
+ bool MinSuccess = DAG->addEdge(Wmmas.begin()[MinDist].first, SDep(LI.SU, SDep::Artificial));
+
+ // Maximum distance edge
+ // Load scheduled after this point
+ bool MaxSuccess = true;
+ std::optional<unsigned> MaxDist = DeadBy[MinDist];
+ if (MaxDist && *MaxDist < MinDist)
+ MaxSuccess = DAG->addEdge(LI.SU, SDep(Wmmas.begin()[*MaxDist].first, SDep::Artificial));
+
+ LLVM_DEBUG(dbgs() << "load SU" << LI.SU->NodeNum << " Win=[" << (MaxDist ? *MaxDist : -1) << ","
+ << MinDist << "] MinPos=" << LI.MinPos << " MaxPos=" << LI.MaxPos
+ << (MinSuccess && MaxSuccess ? "\n" : " (edge REJECTED: cycle)\n"));
+ }
+
}
} // end namespace
>From f53deeaa2e17de2227df87d5c7ea441467d5a9c8 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Thu, 11 Jun 2026 19:34:03 +0000
Subject: [PATCH 4/4] MIR test
---
.../AMDGPU/sched-wmma-ds-load-window.mir | 376 ++++++++++++++++++
1 file changed, 376 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
diff --git a/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir b/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
new file mode 100644
index 0000000000000..6435dc89889e3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-wmma-ds-load-window.mir
@@ -0,0 +1,376 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+#
+# Test for the WMMASchedule DAG mutation on an 8x8 outer product tile
+# (64 DS_READ_B128, 8 A/B fragments, 64 V_WMMA). Without the mutation the
+# scheduler schedules all 64 loads at the beginning of the block; the mutation
+# debunches these loads.
+#
+# XFAIL: *
+
+--- |
+ target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+ target triple = "amdgcn"
+ define amdgpu_kernel void @wmma_op() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx1250" "amdgpu-flat-work-group-size"="1,128" "amdgpu-waves-per-eu"="1,1" }
+...
+---
+name: wmma_op
+alignment: 4
+tracksRegLiveness: true
+isSSA: false
+noPhis: true
+machineFunctionInfo:
+ isEntryFunction: true
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+ occupancy: 1
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: wmma_op
+ ; CHECK: V_WMMA
+ ; CHECK-NEXT: V_WMMA
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: V_WMMA
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: DS_READ_B128
+ ; CHECK-NEXT: V_WMMA
+ successors: %bb.1(0x80000000)
+ %ptra:vgpr_32 = IMPLICIT_DEF
+ %ptrb:vgpr_32 = IMPLICIT_DEF
+ %scale:vreg_64_lo256_align2 = IMPLICIT_DEF
+ %vo:vgpr_32 = IMPLICIT_DEF
+ %rsrc:sgpr_128 = IMPLICIT_DEF
+ %500:vreg_256_align2 = IMPLICIT_DEF
+ %501:vreg_256_align2 = IMPLICIT_DEF
+ %502:vreg_256_align2 = IMPLICIT_DEF
+ %503:vreg_256_align2 = IMPLICIT_DEF
+ %504:vreg_256_align2 = IMPLICIT_DEF
+ %505:vreg_256_align2 = IMPLICIT_DEF
+ %506:vreg_256_align2 = IMPLICIT_DEF
+ %507:vreg_256_align2 = IMPLICIT_DEF
+ %508:vreg_256_align2 = IMPLICIT_DEF
+ %509:vreg_256_align2 = IMPLICIT_DEF
+ %510:vreg_256_align2 = IMPLICIT_DEF
+ %511:vreg_256_align2 = IMPLICIT_DEF
+ %512:vreg_256_align2 = IMPLICIT_DEF
+ %513:vreg_256_align2 = IMPLICIT_DEF
+ %514:vreg_256_align2 = IMPLICIT_DEF
+ %515:vreg_256_align2 = IMPLICIT_DEF
+ %516:vreg_256_align2 = IMPLICIT_DEF
+ %517:vreg_256_align2 = IMPLICIT_DEF
+ %518:vreg_256_align2 = IMPLICIT_DEF
+ %519:vreg_256_align2 = IMPLICIT_DEF
+ %520:vreg_256_align2 = IMPLICIT_DEF
+ %521:vreg_256_align2 = IMPLICIT_DEF
+ %522:vreg_256_align2 = IMPLICIT_DEF
+ %523:vreg_256_align2 = IMPLICIT_DEF
+ %524:vreg_256_align2 = IMPLICIT_DEF
+ %525:vreg_256_align2 = IMPLICIT_DEF
+ %526:vreg_256_align2 = IMPLICIT_DEF
+ %527:vreg_256_align2 = IMPLICIT_DEF
+ %528:vreg_256_align2 = IMPLICIT_DEF
+ %529:vreg_256_align2 = IMPLICIT_DEF
+ %530:vreg_256_align2 = IMPLICIT_DEF
+ %531:vreg_256_align2 = IMPLICIT_DEF
+ %532:vreg_256_align2 = IMPLICIT_DEF
+ %533:vreg_256_align2 = IMPLICIT_DEF
+ %534:vreg_256_align2 = IMPLICIT_DEF
+ %535:vreg_256_align2 = IMPLICIT_DEF
+ %536:vreg_256_align2 = IMPLICIT_DEF
+ %537:vreg_256_align2 = IMPLICIT_DEF
+ %538:vreg_256_align2 = IMPLICIT_DEF
+ %539:vreg_256_align2 = IMPLICIT_DEF
+ %540:vreg_256_align2 = IMPLICIT_DEF
+ %541:vreg_256_align2 = IMPLICIT_DEF
+ %542:vreg_256_align2 = IMPLICIT_DEF
+ %543:vreg_256_align2 = IMPLICIT_DEF
+ %544:vreg_256_align2 = IMPLICIT_DEF
+ %545:vreg_256_align2 = IMPLICIT_DEF
+ %546:vreg_256_align2 = IMPLICIT_DEF
+ %547:vreg_256_align2 = IMPLICIT_DEF
+ %548:vreg_256_align2 = IMPLICIT_DEF
+ %549:vreg_256_align2 = IMPLICIT_DEF
+ %550:vreg_256_align2 = IMPLICIT_DEF
+ %551:vreg_256_align2 = IMPLICIT_DEF
+ %552:vreg_256_align2 = IMPLICIT_DEF
+ %553:vreg_256_align2 = IMPLICIT_DEF
+ %554:vreg_256_align2 = IMPLICIT_DEF
+ %555:vreg_256_align2 = IMPLICIT_DEF
+ %556:vreg_256_align2 = IMPLICIT_DEF
+ %557:vreg_256_align2 = IMPLICIT_DEF
+ %558:vreg_256_align2 = IMPLICIT_DEF
+ %559:vreg_256_align2 = IMPLICIT_DEF
+ %560:vreg_256_align2 = IMPLICIT_DEF
+ %561:vreg_256_align2 = IMPLICIT_DEF
+ %562:vreg_256_align2 = IMPLICIT_DEF
+ %563:vreg_256_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ undef %300.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %300.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %301.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %301.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %302.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %302.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %303.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %303.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %304.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %304.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %305.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %305.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %306.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %306.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %307.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %307.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptra, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %400.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %400.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %401.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %401.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %402.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %402.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %403.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %403.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %404.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %404.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %405.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %405.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %406.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %406.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ undef %407.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 64, 0, implicit $exec :: (load (s128), addrspace 3)
+ %407.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %ptrb, 96, 0, implicit $exec :: (load (s128), addrspace 3)
+ early-clobber %500:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %400, 8, %500, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %501:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %401, 8, %501, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %502:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %402, 8, %502, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %503:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %403, 8, %503, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %504:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %404, 8, %504, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %505:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %405, 8, %505, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %506:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %406, 8, %506, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %507:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %300, %407, 8, %507, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %508:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %400, 8, %508, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %509:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %401, 8, %509, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %510:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %402, 8, %510, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %511:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %403, 8, %511, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %512:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %404, 8, %512, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %513:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %405, 8, %513, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %514:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %406, 8, %514, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %515:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %301, %407, 8, %515, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %516:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %400, 8, %516, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %517:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %401, 8, %517, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %518:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %402, 8, %518, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %519:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %403, 8, %519, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %520:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %404, 8, %520, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %521:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %405, 8, %521, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %522:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %406, 8, %522, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %523:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %302, %407, 8, %523, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %524:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %400, 8, %524, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %525:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %401, 8, %525, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %526:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %402, 8, %526, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %527:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %403, 8, %527, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %528:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %404, 8, %528, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %529:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %405, 8, %529, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %530:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %406, 8, %530, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %531:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %303, %407, 8, %531, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %532:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %400, 8, %532, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %533:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %401, 8, %533, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %534:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %402, 8, %534, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %535:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %403, 8, %535, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %536:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %404, 8, %536, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %537:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %405, 8, %537, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %538:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %406, 8, %538, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %539:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %304, %407, 8, %539, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %540:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %400, 8, %540, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %541:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %401, 8, %541, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %542:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %402, 8, %542, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %543:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %403, 8, %543, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %544:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %404, 8, %544, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %545:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %405, 8, %545, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %546:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %406, 8, %546, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %547:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %305, %407, 8, %547, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %548:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %400, 8, %548, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %549:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %401, 8, %549, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %550:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %402, 8, %550, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %551:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %403, 8, %551, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %552:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %404, 8, %552, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %553:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %405, 8, %553, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %554:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %406, 8, %554, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %555:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %306, %407, 8, %555, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %556:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %400, 8, %556, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %557:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %401, 8, %557, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %558:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %402, 8, %558, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %559:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %403, 8, %559, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %560:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %404, 8, %560, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %561:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %405, 8, %561, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %562:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %406, 8, %562, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %563:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %307, %407, 8, %563, %scale.sub0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %500.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %500.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %501.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %501.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %502.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %502.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %503.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %503.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %504.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %504.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %505.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %505.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %506.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %506.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %507.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %507.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %508.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %508.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %509.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %509.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %510.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %510.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %511.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %511.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %512.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %512.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %513.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %513.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %514.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %514.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %515.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %515.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %516.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %516.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %517.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %517.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %518.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %518.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %519.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %519.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %520.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %520.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %521.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %521.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %522.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %522.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %523.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %523.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %524.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %524.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %525.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %525.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %526.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %526.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %527.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %527.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %528.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %528.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %529.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %529.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %530.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %530.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %531.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %531.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %532.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %532.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %533.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %533.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %534.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %534.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %535.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %535.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %536.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %536.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %537.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %537.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %538.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %538.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %539.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %539.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %540.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %540.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %541.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %541.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %542.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %542.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %543.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %543.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %544.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %544.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %545.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %545.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %546.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %546.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %547.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %547.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %548.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %548.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %549.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %549.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %550.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %550.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %551.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %551.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %552.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %552.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %553.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %553.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %554.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %554.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %555.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %555.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %556.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %556.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %557.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %557.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %558.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %558.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %559.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %559.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %560.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %560.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %561.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %561.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %562.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %562.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %563.sub0_sub1_sub2_sub3, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact %563.sub4_sub5_sub6_sub7, %vo, %rsrc, $sgpr_null, 0, 0, 0, implicit $exec :: (store (s128), addrspace 8)
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list