[llvm] [AMDGPU] Model ordered XDL writes in expert scheduling (PR #218326)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 23:16:21 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Austin Kerbow (kerbowa)
<details>
<summary>Changes</summary>
Treat qualifying XDL WMMA as VA_VDST fences so expert scheduling can use nonzero waits after out-of-order event mixes.
An XDL instruction with a destination register of 8 or more VGPRs decrements VA_VDST after all instructions that were issued earlier.
This offers a significant benefit in compute bound kernels since we are not required to pessimistically wait for WMMAs to write their results when interleaving LDS.
```
VALU def r0
TRANS
VALU
TRANS
WMMA
va_vdst(1) # previously this would require va_vdst(0)
DS_LOAD use r0
```
---
Patch is 55.63 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218326.diff
7 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp (+4-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def (+2-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h (+3-3)
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+93-27)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+14)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.h (+4)
- (added) llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir (+511)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
index 946aca72e2cb1..639c386a12dcc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
@@ -27,8 +27,11 @@ static HWEvents getExpertSchedulingEventType(const MachineInstr &Inst,
// out-of-order with respect to each other, so each of these classes
// has its own event.
- if (TII.isXDL(Inst))
+ if (TII.isXDL(Inst)) {
+ if (TII.isVAVDSTOrderedXDL(Inst))
+ return HWEvents::VGPR_XDL_READ | HWEvents::VGPR_XDL_ORDERED_WRITE;
return HWEvents::VGPR_XDL_READ | HWEvents::VGPR_XDL_WRITE;
+ }
if (TII.isTRANS(Inst))
return HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_TRANS_WRITE;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
index dc97c0b8ae1f7..6af3786098187 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
@@ -51,8 +51,9 @@ AMDGPU_HW_EVENT(VGPR_FLAT_READ, 28) /* read VGPR source in FLAT */
AMDGPU_HW_EVENT(VGPR_VMEM_READ, 29) /* read VGPR source in other VMEM */
AMDGPU_HW_EVENT(ASYNC_ACCESS, 30) /* access that uses ASYNC_CNT */
AMDGPU_HW_EVENT(TENSOR_ACCESS, 31) /* access that uses TENSOR_CNT */
+AMDGPU_HW_EVENT(VGPR_XDL_ORDERED_WRITE, 32) /* write VGPR dest in ordered XDL VALU */
-AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
+AMDGPU_LAST_HW_EVENT(VGPR_XDL_ORDERED_WRITE)
// clang-format on
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
index eb25206b5ee04..d8a2b04e9aff3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
@@ -49,11 +49,11 @@ namespace AMDGPU {
/// - Constructor ensures undefined bits cannot be set.
class HWEvents {
public:
- using value_type = uint32_t;
+ using value_type = uint64_t;
enum : value_type {
NONE = 0,
-#define AMDGPU_HW_EVENT(X, V) X = (1u << V),
+#define AMDGPU_HW_EVENT(X, V) X = (value_type(1) << V),
#define AMDGPU_LAST_HW_EVENT(X) HWEVENT_LAST_EVENT = X,
#include "AMDGPUHWEvents.def"
@@ -80,7 +80,7 @@ class HWEvents {
HWEvents operator*() const {
// Return only rightmost (least significant) bit set.
- return Cur ? (Cur & (1 << countr_zero(Cur))) : 0;
+ return Cur ? (Cur & (HWEvents::value_type(1) << countr_zero(Cur))) : 0;
}
const_iterator &operator++() {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index d73565481e9c9..b0ee0e89717e3 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -301,7 +301,8 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
- HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
+ HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE |
+ HWEvents::VGPR_XDL_ORDERED_WRITE,
HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
HWEvents::VGPR_VMEM_READ};
@@ -533,6 +534,11 @@ class WaitcntBrackets {
return It != VMem.end() ? It->second.Scores[T] : 0;
}
+ unsigned getNumOrderedXDLsAfterWrite(VMEMID TID) const {
+ auto It = VMem.find(TID);
+ return It != VMem.end() ? It->second.NumOrderedXDLsAfterWrite : 0;
+ }
+
public:
// Do some internal consistency checks.
void verify() {
@@ -670,8 +676,11 @@ class WaitcntBrackets {
using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
+ /// \p IndependentWaitBound is a wait-count value proven safe independently
+ /// of the score and event-class ordering.
void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
- AMDGPU::Waitcnt &Wait) const;
+ AMDGPU::Waitcnt &Wait,
+ unsigned IndependentWaitBound = 0) const;
static bool mergeScore(const MergeInfo &M, unsigned &Score,
unsigned OtherScore);
@@ -707,8 +716,12 @@ class WaitcntBrackets {
if (Reg == AMDGPU::SCC) {
SCCScore = Val;
} else if (TRI.isVectorRegister(Context->MRI, Reg)) {
- for (MCRegUnit RU : regunits(Reg))
- VMem[toVMEMID(RU)].Scores[T] = Val;
+ for (MCRegUnit RU : regunits(Reg)) {
+ auto &Info = VMem[toVMEMID(RU)];
+ Info.Scores[T] = Val;
+ if (T == AMDGPU::VA_VDST_WR)
+ Info.NumOrderedXDLsAfterWrite = 0;
+ }
} else if (TRI.isSGPRReg(Context->MRI, Reg)) {
for (MCRegUnit RU : regunits(Reg))
SGPRs[RU].get(T) = Val;
@@ -718,7 +731,10 @@ class WaitcntBrackets {
}
void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
- VMem[TID].Scores[T] = Val;
+ auto &Info = VMem[TID];
+ Info.Scores[T] = Val;
+ if (T == AMDGPU::VA_VDST_WR)
+ Info.NumOrderedXDLsAfterWrite = 0;
}
void setScoreByOperand(const MachineOperand &Op,
@@ -751,6 +767,9 @@ class WaitcntBrackets {
struct VMEMInfo {
// Scores for all instruction counters. Zero-initialized.
CounterValueArray Scores{};
+ // Later ordered XDL writes that must remain outstanding while this
+ // VA_VDST write is pending.
+ unsigned NumOrderedXDLsAfterWrite = 0;
// For VGPRs, we need to track an additional fine-grained set of pending
// events.
HWEvents VGPRPendingEvents;
@@ -884,7 +903,7 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
Increment = 2;
}
unsigned CurrScore = UB + Increment;
- if (CurrScore == 0)
+ if (CurrScore <= UB)
report_fatal_error("InsertWaitcnt score wraparound");
// PendingEvents and ScoreUB need to be update regardless if this event
// changes the score of a register or not.
@@ -892,6 +911,17 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
PendingEvents |= E;
setScoreUB(T, CurrScore);
+ if (E == HWEvents::VGPR_XDL_ORDERED_WRITE) {
+ // While an earlier write is pending, every later ordered XDL write must
+ // also be pending. Count those writes as a per-dependency wait bound.
+ for (auto &Entry : VMem) {
+ auto &Info = Entry.second;
+ if (Info.Scores[AMDGPU::VA_VDST_WR] > getScoreLB(AMDGPU::VA_VDST_WR) &&
+ Info.NumOrderedXDLsAfterWrite < getLimit(AMDGPU::VA_VDST_WR) - 1)
+ ++Info.NumOrderedXDLsAfterWrite;
+ }
+ }
+
const SIRegisterInfo &TRI = Context->TRI;
const MachineRegisterInfo &MRI = Context->MRI;
const SIInstrInfo &TII = Context->TII;
@@ -1344,9 +1374,9 @@ void WaitcntBrackets::purgeEmptyTrackingData() {
SGPRs.remove_if([](const auto &P) { return P.second.empty(); });
}
-void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
- unsigned ScoreToWait,
- AMDGPU::Waitcnt &Wait) const {
+void WaitcntBrackets::determineWaitForScore(
+ AMDGPU::InstCounterType T, unsigned ScoreToWait, AMDGPU::Waitcnt &Wait,
+ unsigned IndependentWaitBound) const {
const unsigned LB = getScoreLB(T);
const unsigned UB = getScoreUB(T);
@@ -1358,16 +1388,11 @@ void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
// waitcnt and the target can report early completion, then we need
// to force a waitcnt 0.
Wait.add(T, 0);
- } else if (counterOutOfOrder(T)) {
- // Counter can get decremented out-of-order when there
- // are multiple types event in the bracket. Also emit an s_wait counter
- // with a conservative value of 0 for the counter.
- Wait.add(T, 0);
} else {
- // If a counter has been maxed out avoid overflow by waiting for
- // MAX(CounterType) - 1 instead.
- unsigned NeededWait = std::min(UB - ScoreToWait, getLimit(T) - 1);
- Wait.add(T, NeededWait);
+ unsigned ExactWaitBound = 0;
+ if (!counterOutOfOrder(T))
+ ExactWaitBound = std::min(UB - ScoreToWait, getLimit(T) - 1);
+ Wait.add(T, std::max(ExactWaitBound, IndependentWaitBound));
}
}
}
@@ -1439,8 +1464,12 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
AMDGPU::isHi16Reg(Reg, Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
AMDGPU::Waitcnt Wait;
- for (MCRegUnit RU : regunits(OtherHalf))
- determineWaitForScore(T, getVMemScore(toVMEMID(RU), T), Wait);
+ for (MCRegUnit RU : regunits(OtherHalf)) {
+ VMEMID ID = toVMEMID(RU);
+ unsigned IndependentWaitBound =
+ T == AMDGPU::VA_VDST_WR ? getNumOrderedXDLsAfterWrite(ID) : 0;
+ determineWaitForScore(T, getVMemScore(ID, T), Wait, IndependentWaitBound);
+ }
// No wait on otherhalf
if (!Wait.hasWait())
@@ -1469,10 +1498,15 @@ void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
bool IsVGPR = Context->TRI.isVectorRegister(Context->MRI, Reg);
if (IsVGPR)
Reg = determineVGPR16Dependency(MI, T, Reg);
- for (MCRegUnit RU : regunits(Reg))
- determineWaitForScore(
- T, IsVGPR ? getVMemScore(toVMEMID(RU), T) : getSGPRScore(RU, T),
- Wait);
+ for (MCRegUnit RU : regunits(Reg)) {
+ VMEMID ID = toVMEMID(RU);
+ unsigned IndependentWaitBound = IsVGPR && T == AMDGPU::VA_VDST_WR
+ ? getNumOrderedXDLsAfterWrite(ID)
+ : 0;
+ determineWaitForScore(T,
+ IsVGPR ? getVMemScore(ID, T) : getSGPRScore(RU, T),
+ Wait, IndependentWaitBound);
+ }
}
}
@@ -1519,9 +1553,21 @@ void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
if (Count >= UB)
return;
if (Count != 0) {
- if (counterOutOfOrder(T))
- return;
- setScoreLB(T, std::max(getScoreLB(T), UB - Count));
+ const unsigned NewLB = std::max(getScoreLB(T), UB - Count);
+ if (counterOutOfOrder(T)) {
+ if (T != AMDGPU::VA_VDST_RD && T != AMDGPU::VA_VDST_WR)
+ return;
+
+ // VA_VDST_RD and VA_VDST_WR share a score range. Keep unresolved
+ // dependencies above the new lower bound when either side is out of
+ // order.
+ for (auto &[ID, Info] : VMem) {
+ unsigned &Score = Info.Scores[T];
+ if (Score > getScoreLB(T) && Score <= NewLB)
+ Score = NewLB + 1;
+ }
+ }
+ setScoreLB(T, NewLB);
} else {
setScoreLB(T, UB);
PendingEvents -= Context->getWaitEvents(T);
@@ -2891,6 +2937,26 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
+ if (T == AMDGPU::VA_VDST_WR) {
+ // A fence count is usable only when every path carrying the dependency
+ // has at least that many later ordered XDL writes.
+ for (auto &[RegID, Info] : VMem) {
+ auto OtherIt = Other.VMem.find(RegID);
+ unsigned OtherScore =
+ OtherIt == Other.VMem.end() ? 0 : OtherIt->second.Scores[T];
+ bool MyPending = Info.Scores[T] > M.OldLB;
+ bool OtherPending = OtherScore > M.OtherLB;
+ unsigned OtherCount =
+ OtherPending ? OtherIt->second.NumOrderedXDLsAfterWrite : 0;
+ unsigned NewCount =
+ MyPending && OtherPending
+ ? std::min(Info.NumOrderedXDLsAfterWrite, OtherCount)
+ : (MyPending ? Info.NumOrderedXDLsAfterWrite : OtherCount);
+ StrictDom |= MyPending && NewCount < Info.NumOrderedXDLsAfterWrite;
+ Info.NumOrderedXDLsAfterWrite = NewCount;
+ }
+ }
+
for (auto &[RegID, Info] : VMem)
StrictDom |= mergeScore(M, Info.Scores[T], Other.getVMemScore(RegID, T));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index b2ae51a1b85f7..c67b5ce8eeb9f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11753,3 +11753,17 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
return AMDGPU::getMAIIsGFX940XDL(Opcode);
}
+
+bool SIInstrInfo::isVAVDSTOrderedXDL(const MachineInstr &MI) const {
+ constexpr unsigned MinOrderedXDLVAVDSTVGPRs = 8;
+ if (!AMDGPU::isGFX1250(ST) || !isXDL(MI))
+ return false;
+
+ int DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst);
+ if (DstIdx < 0)
+ return false;
+
+ const TargetRegisterClass *DstRC = getRegClass(MI.getDesc(), DstIdx);
+ return DstRC && RI.isVGPRClass(DstRC) &&
+ RI.getRegSizeInBits(*DstRC) >= MinOrderedXDLVAVDSTVGPRs * 32;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 6c7b2d7d2279e..62067217b0687 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1015,6 +1015,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool isXDL(const MachineInstr &MI) const;
+ /// On gfx1250, XDL VA_VDST decrements are ordered after earlier decrements
+ /// when the destination spans at least eight sequential VGPRs.
+ bool isVAVDSTOrderedXDL(const MachineInstr &MI) const;
+
static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }
static bool isLDSDIR(const MachineInstr &MI) {
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir
new file mode 100644
index 0000000000000..9059ab64cbdd9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-vavdst-ordered-xdl.mir
@@ -0,0 +1,511 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs -amdgpu-expert-scheduling-mode -run-pass si-insert-waitcnts %s -o - | FileCheck %s
+
+---
+name: mixed_events_use_ordered_xdl_fence_bound
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr28, $vgpr29, $vgpr31, $vgpr33, $vgpr35, $vgpr40, $vgpr41, $vgpr42, $vgpr43
+
+ ; CHECK-LABEL: name: mixed_events_use_ordered_xdl_fence_bound
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr28, $vgpr29, $vgpr31, $vgpr33, $vgpr35, $vgpr40, $vgpr41, $vgpr42, $vgpr43
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: $vgpr32 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr33 = V_DOT2C_F32_F16_e32 $vgpr28, $vgpr29, $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr34 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr35, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr32 = V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ $vgpr33 = V_DOT2C_F32_F16_e32 $vgpr28, $vgpr29, $vgpr33, implicit $mode, implicit $exec
+ $vgpr40_vgpr41_vgpr42_vgpr43 = V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, 0, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ $vgpr34 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr35, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: ordered_swmmac
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr4, $vgpr5, $vgpr24, $vgpr28, $vgpr29, $vgpr31, $vgpr32, $vgpr33, $vgpr34, $vgpr35, $vgpr36, $vgpr37, $vgpr38, $vgpr39, $vgpr40, $vgpr41, $vgpr42, $vgpr43, $vgpr44, $vgpr45, $vgpr46, $vgpr47, $vgpr48, $vgpr49, $vgpr50, $vgpr51, $vgpr52, $vgpr53, $vgpr54, $vgpr55, $vgpr56, $vgpr57, $vgpr58, $vgpr59, $vgpr60, $vgpr61, $vgpr62, $vgpr63
+
+ ; CHECK-LABEL: name: ordered_swmmac
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr24, $vgpr28, $vgpr29, $vgpr31, $vgpr32, $vgpr33, $vgpr34, $vgpr35, $vgpr36, $vgpr37, $vgpr38, $vgpr39, $vgpr40, $vgpr41, $vgpr42, $vgpr43, $vgpr44, $vgpr45, $vgpr46, $vgpr47, $vgpr48, $vgpr49, $vgpr50, $vgpr51, $vgpr52, $vgpr53, $vgpr54, $vgpr55, $vgpr56, $vgpr57, $vgpr58, $vgpr59, $vgpr60, $vgpr61, $vgpr62, $vgpr63
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ $vgpr25 = V_ADD_F32_e32 $vgpr28, $vgpr29, implicit $mode, implicit $exec
+ DS_WRITE_B32_gfx9 $vgpr24, $vgpr30, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: eight_vgpr_destination_ordered_at_ub
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr31
+
+ ; CHECK-LABEL: name: eight_vgpr_destination_ordered_at_ub
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_MOV_B32_e32 $vgpr31, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr1...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218326
More information about the llvm-commits
mailing list