[llvm] [AMDGPU] Model ordered XDL writes in expert scheduling (PR #218326)
Austin Kerbow via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 23:39:40 PDT 2026
https://github.com/kerbowa updated https://github.com/llvm/llvm-project/pull/218326
>From f2c18d8eac0282cff1b01594efcc58fa9cf98e9d Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow at amd.com>
Date: Sun, 23 Aug 2026 21:29:37 -0500
Subject: [PATCH] [AMDGPU] Model ordered XDL writes in expert scheduling
Treat qualifying XDL WMMA as VA_VDST fences so expert scheduling can use
nonzero waits after out-of-order event mixes.
An XDL instruction with a destination register of 8 or more VGPRs decrements
VA_VDST after all instructions that were issued earlier.
This offers a significant benefit in compute bound kernels since we are not
required to pessimistically wait for WMMAs to write their results when
interleaving LDS.
```
VALU def r0
TRANS
VALU
TRANS
WMMA
va_vdst(1) # previously this would require va_vdst(0)
DS_LOAD use r0
```
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 139 ++++--
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 +
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +
.../AMDGPU/waitcnt-vavdst-ordered-xdl.mir | 445 ++++++++++++++++++
4 files changed, 575 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index d73565481e9c9..5c7b4253a223f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -533,6 +533,11 @@ class WaitcntBrackets {
return It != VMem.end() ? It->second.Scores[T] : 0;
}
+ unsigned getNumOrderedXDLsAfterWrite(VMEMID TID) const {
+ auto It = VMem.find(TID);
+ return It != VMem.end() ? It->second.NumOrderedXDLsAfterWrite : 0;
+ }
+
public:
// Do some internal consistency checks.
void verify() {
@@ -670,8 +675,12 @@ class WaitcntBrackets {
using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
+ std::optional<unsigned> getWaitForScore(AMDGPU::InstCounterType T,
+ unsigned Score) const;
void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
AMDGPU::Waitcnt &Wait) const;
+ void determineWaitForVGPRScore(AMDGPU::InstCounterType T, VMEMID ID,
+ AMDGPU::Waitcnt &Wait) const;
static bool mergeScore(const MergeInfo &M, unsigned &Score,
unsigned OtherScore);
@@ -707,8 +716,12 @@ class WaitcntBrackets {
if (Reg == AMDGPU::SCC) {
SCCScore = Val;
} else if (TRI.isVectorRegister(Context->MRI, Reg)) {
- for (MCRegUnit RU : regunits(Reg))
- VMem[toVMEMID(RU)].Scores[T] = Val;
+ for (MCRegUnit RU : regunits(Reg)) {
+ auto &Info = VMem[toVMEMID(RU)];
+ Info.Scores[T] = Val;
+ if (T == AMDGPU::VA_VDST_WR)
+ Info.NumOrderedXDLsAfterWrite = 0;
+ }
} else if (TRI.isSGPRReg(Context->MRI, Reg)) {
for (MCRegUnit RU : regunits(Reg))
SGPRs[RU].get(T) = Val;
@@ -718,7 +731,10 @@ class WaitcntBrackets {
}
void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
- VMem[TID].Scores[T] = Val;
+ auto &Info = VMem[TID];
+ Info.Scores[T] = Val;
+ if (T == AMDGPU::VA_VDST_WR)
+ Info.NumOrderedXDLsAfterWrite = 0;
}
void setScoreByOperand(const MachineOperand &Op,
@@ -751,6 +767,9 @@ class WaitcntBrackets {
struct VMEMInfo {
// Scores for all instruction counters. Zero-initialized.
CounterValueArray Scores{};
+ // Later ordered XDL writes that must remain outstanding while this
+ // VA_VDST write is pending.
+ unsigned NumOrderedXDLsAfterWrite = 0;
// For VGPRs, we need to track an additional fine-grained set of pending
// events.
HWEvents VGPRPendingEvents;
@@ -884,7 +903,8 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
Increment = 2;
}
unsigned CurrScore = UB + Increment;
- if (CurrScore == 0)
+ // Increment can be two, so unsigned overflow may wrap to one instead of zero.
+ if (CurrScore <= UB)
report_fatal_error("InsertWaitcnt score wraparound");
// PendingEvents and ScoreUB need to be update regardless if this event
// changes the score of a register or not.
@@ -892,6 +912,17 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
PendingEvents |= E;
setScoreUB(T, CurrScore);
+ if (E == HWEvents::VGPR_XDL_WRITE && Context->TII.isVAVDSTOrderedXDL(Inst)) {
+ // While an earlier write is pending, every later ordered XDL write must
+ // also be pending. Count those writes as a per-dependency wait bound.
+ for (auto &Entry : VMem) {
+ auto &Info = Entry.second;
+ if (Info.Scores[AMDGPU::VA_VDST_WR] > getScoreLB(AMDGPU::VA_VDST_WR) &&
+ Info.NumOrderedXDLsAfterWrite < getLimit(AMDGPU::VA_VDST_WR) - 1)
+ ++Info.NumOrderedXDLsAfterWrite;
+ }
+ }
+
const SIRegisterInfo &TRI = Context->TRI;
const MachineRegisterInfo &MRI = Context->MRI;
const SIInstrInfo &TII = Context->TII;
@@ -1344,9 +1375,9 @@ void WaitcntBrackets::purgeEmptyTrackingData() {
SGPRs.remove_if([](const auto &P) { return P.second.empty(); });
}
-void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
- unsigned ScoreToWait,
- AMDGPU::Waitcnt &Wait) const {
+std::optional<unsigned>
+WaitcntBrackets::getWaitForScore(AMDGPU::InstCounterType T,
+ unsigned ScoreToWait) const {
const unsigned LB = getScoreLB(T);
const unsigned UB = getScoreUB(T);
@@ -1357,19 +1388,35 @@ void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
// If there is a pending FLAT operation, and this is a VMem or LGKM
// waitcnt and the target can report early completion, then we need
// to force a waitcnt 0.
- Wait.add(T, 0);
- } else if (counterOutOfOrder(T)) {
- // Counter can get decremented out-of-order when there
- // are multiple types event in the bracket. Also emit an s_wait counter
- // with a conservative value of 0 for the counter.
- Wait.add(T, 0);
- } else {
- // If a counter has been maxed out avoid overflow by waiting for
- // MAX(CounterType) - 1 instead.
- unsigned NeededWait = std::min(UB - ScoreToWait, getLimit(T) - 1);
- Wait.add(T, NeededWait);
+ return 0;
}
+ if (!counterOutOfOrder(T))
+ return std::min(UB - ScoreToWait, getLimit(T) - 1);
+ return 0;
}
+ return std::nullopt;
+}
+
+void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
+ unsigned ScoreToWait,
+ AMDGPU::Waitcnt &Wait) const {
+ if (std::optional<unsigned> Count = getWaitForScore(T, ScoreToWait))
+ Wait.add(T, *Count);
+}
+
+void WaitcntBrackets::determineWaitForVGPRScore(AMDGPU::InstCounterType T,
+ VMEMID ID,
+ AMDGPU::Waitcnt &Wait) const {
+ std::optional<unsigned> Count = getWaitForScore(T, getVMemScore(ID, T));
+ if (!Count)
+ return;
+
+ // The ordered-XDL count is a wait bound independent of exact event-class
+ // ordering. Combine the bounds for this register unit before Wait.add()
+ // takes the minimum across all required register-unit dependencies.
+ if (T == AMDGPU::VA_VDST_WR)
+ *Count = std::max(*Count, getNumOrderedXDLsAfterWrite(ID));
+ Wait.add(T, *Count);
}
AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
@@ -1439,8 +1486,10 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
AMDGPU::isHi16Reg(Reg, Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
AMDGPU::Waitcnt Wait;
- for (MCRegUnit RU : regunits(OtherHalf))
- determineWaitForScore(T, getVMemScore(toVMEMID(RU), T), Wait);
+ for (MCRegUnit RU : regunits(OtherHalf)) {
+ VMEMID ID = toVMEMID(RU);
+ determineWaitForVGPRScore(T, ID, Wait);
+ }
// No wait on otherhalf
if (!Wait.hasWait())
@@ -1469,10 +1518,14 @@ void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
bool IsVGPR = Context->TRI.isVectorRegister(Context->MRI, Reg);
if (IsVGPR)
Reg = determineVGPR16Dependency(MI, T, Reg);
- for (MCRegUnit RU : regunits(Reg))
- determineWaitForScore(
- T, IsVGPR ? getVMemScore(toVMEMID(RU), T) : getSGPRScore(RU, T),
- Wait);
+ for (MCRegUnit RU : regunits(Reg)) {
+ VMEMID ID = toVMEMID(RU);
+ if (!IsVGPR) {
+ determineWaitForScore(T, getSGPRScore(RU, T), Wait);
+ continue;
+ }
+ determineWaitForVGPRScore(T, ID, Wait);
+ }
}
}
@@ -1519,9 +1572,21 @@ void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
if (Count >= UB)
return;
if (Count != 0) {
- if (counterOutOfOrder(T))
- return;
- setScoreLB(T, std::max(getScoreLB(T), UB - Count));
+ const unsigned NewLB = std::max(getScoreLB(T), UB - Count);
+ if (counterOutOfOrder(T)) {
+ if (T != AMDGPU::VA_VDST_RD && T != AMDGPU::VA_VDST_WR)
+ return;
+
+ // VA_VDST_RD and VA_VDST_WR share a score range. Keep unresolved
+ // dependencies above the new lower bound when either side is out of
+ // order.
+ for (auto &[ID, Info] : VMem) {
+ unsigned &Score = Info.Scores[T];
+ if (Score > getScoreLB(T) && Score <= NewLB)
+ Score = NewLB + 1;
+ }
+ }
+ setScoreLB(T, NewLB);
} else {
setScoreLB(T, UB);
PendingEvents -= Context->getWaitEvents(T);
@@ -2891,6 +2956,26 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
+ if (T == AMDGPU::VA_VDST_WR) {
+ // A fence count is usable only when every path carrying the dependency
+ // has at least that many later ordered XDL writes.
+ for (auto &[RegID, Info] : VMem) {
+ auto OtherIt = Other.VMem.find(RegID);
+ unsigned OtherScore =
+ OtherIt == Other.VMem.end() ? 0 : OtherIt->second.Scores[T];
+ bool MyPending = Info.Scores[T] > M.OldLB;
+ bool OtherPending = OtherScore > M.OtherLB;
+ unsigned OtherCount =
+ OtherPending ? OtherIt->second.NumOrderedXDLsAfterWrite : 0;
+ unsigned NewCount =
+ MyPending && OtherPending
+ ? std::min(Info.NumOrderedXDLsAfterWrite, OtherCount)
+ : (MyPending ? Info.NumOrderedXDLsAfterWrite : OtherCount);
+ StrictDom |= MyPending && NewCount < Info.NumOrderedXDLsAfterWrite;
+ Info.NumOrderedXDLsAfterWrite = NewCount;
+ }
+ }
+
for (auto &[RegID, Info] : VMem)
StrictDom |= mergeScore(M, Info.Scores[T], Other.getVMemScore(RegID, T));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f5000ff582282..6a7cda3bd1a77 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11831,3 +11831,17 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
return AMDGPU::getMAIIsGFX940XDL(Opcode);
}
+
+bool SIInstrInfo::isVAVDSTOrderedXDL(const MachineInstr &MI) const {
+ constexpr unsigned MinOrderedXDLVAVDSTVGPRs = 8;
+ if (!AMDGPU::isGFX1250(ST) || !isXDL(MI))
+ return false;
+
+ int DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst);
+ if (DstIdx < 0)
+ return false;
+
+ const TargetRegisterClass *DstRC = getRegClass(MI.getDesc(), DstIdx);
+ return DstRC && RI.isVGPRClass(DstRC) &&
+ RI.getRegSizeInBits(*DstRC) >= MinOrderedXDLVAVDSTVGPRs * 32;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index a62a753b41b08..6b98d8757e370 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1018,6 +1018,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool isXDL(const MachineInstr &MI) const;
+ /// On gfx1250, XDL VA_VDST decrements are ordered after earlier decrements
+ /// when the destination spans at least eight sequential VGPRs.
+ bool isVAVDSTOrderedXDL(const MachineInstr &MI) const;
+
static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }
static bool isLDSDIR(const MachineInstr &MI) {
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir
new file mode 100644
index 0000000000000..251b16682604e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir
@@ -0,0 +1,445 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50 -verify-machineinstrs -amdgpu-expert-scheduling-mode -run-pass=si-insert-waitcnts %s -o - | FileCheck %s
+
+---
+name: mixed_events_use_ordered_xdl_fence_bound
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: mixed_events_use_ordered_xdl_fence_bound
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: $vgpr32 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr33 = V_DOT2C_F32_F16_e32 $vgpr28, $vgpr29, $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr34 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr35, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr32 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ $vgpr33 = V_DOT2C_F32_F16_e32 $vgpr28, $vgpr29, $vgpr33, implicit $mode, implicit $exec
+ $vgpr40_vgpr41_vgpr42_vgpr43 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, 0, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr34 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr35, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_i32_swmmac_uses_fence_bound
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: ordered_i32_swmmac_uses_fence_bound
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: eight_vgpr_destination_is_ordered
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: eight_vgpr_destination_is_ordered
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: four_vgpr_f16_destination_not_ordered
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: four_vgpr_f16_destination_not_ordered
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr22_vgpr23_vgpr24_vgpr25 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr22_vgpr23_vgpr24_vgpr25, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr27 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr26, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr22_vgpr23_vgpr24_vgpr25 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr22_vgpr23_vgpr24_vgpr25, 0, 0, 0, 0, implicit $exec
+ $vgpr27 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr26, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: narrow_bf16_destination_not_ordered
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: narrow_bf16_destination_not_ordered
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_BF16F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_BF16F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: f32_input_wmma_is_not_xdl
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: f32_input_wmma_is_not_xdl
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr0_vgpr1, $vgpr2_vgpr3, 8, $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_2
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr0_vgpr1, $vgpr2_vgpr3, 8, $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: sixteen_vgpr_destination_is_ordered
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: sixteen_vgpr_destination_is_ordered
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: matrix_scale_increments_vavdst_twice
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: matrix_scale_increments_vavdst_twice
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_2
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr56, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, 0, 0, 0, 0, implicit $exec
+ $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr56, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_wmma_cfg_merge
+tracksRegLiveness: false
+body: |
+ ; CHECK-LABEL: name: ordered_wmma_cfg_merge
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ S_CBRANCH_SCC0 %bb.2, implicit undef $scc
+
+ bb.1:
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.2:
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+
+ bb.3:
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_then_trans_requires_zero
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: ordered_then_trans_requires_zero
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr16, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_then_narrow_xdl_uses_exact_class_bound
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: ordered_then_narrow_xdl_uses_exact_class_bound
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr22_vgpr23_vgpr24_vgpr25 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr22_vgpr23_vgpr24_vgpr25, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr26, $vgpr16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr22_vgpr23_vgpr24_vgpr25 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr22_vgpr23_vgpr24_vgpr25, 0, 0, 0, 0, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr26, $vgpr16, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: partial_wait_reconciles_asymmetric_shared_range
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: partial_wait_reconciles_asymmetric_shared_range
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr24_vgpr25_vgpr26_vgpr27 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr24_vgpr25_vgpr26_vgpr27, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: S_NOP 0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr40, $vgpr16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr24_vgpr25_vgpr26_vgpr27 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr24_vgpr25_vgpr26_vgpr27, 0, 0, 0, 0, implicit $exec
+ S_WAITCNT_DEPCTR .VaVdst_1
+ S_NOP 0
+ DS_WRITE_B32_gfx9 $vgpr40, $vgpr16, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: explicit_zero_prevents_write_resurrection
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: explicit_zero_prevents_write_resurrection
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ S_WAITCNT_DEPCTR .VaVdst_0
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: prior_write_uses_one_sided_fence_bound
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: prior_write_uses_one_sided_fence_bound
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr26 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ $vgpr26 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: fence_destination_with_later_writes_requires_zero
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: fence_destination_with_later_writes_requires_zero
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: $vgpr26 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr26 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr16, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_fence_loop_merge
+tracksRegLiveness: false
+body: |
+ ; CHECK-LABEL: name: ordered_fence_loop_merge
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: early-clobber $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ S_CBRANCH_SCC0 %bb.3, implicit undef $scc
+
+ bb.2:
+ $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.3:
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: read_side_war_remains_conservative
+tracksRegLiveness: false
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: read_side_war_remains_conservative
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr2 = V_ADD_F32_e32 $vgpr0, $vgpr1, implicit $mode, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: $vgpr0 = DS_READ_B32_gfx9 $vgpr24, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr2 = V_ADD_F32_e32 $vgpr0, $vgpr1, implicit $mode, implicit $exec
+ $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ $vgpr0 = DS_READ_B32_gfx9 $vgpr24, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list