[llvm] [RFC][AMDGPU][InsertWaitCnt] Move WaitEventType into separate HWEvent header (PR #202886)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 04:15:52 PDT 2026
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/202886
>From 46a119de421bd6a3733914bf25a665ceaa6dd367 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Tue, 9 Jun 2026 16:54:38 +0200
Subject: [PATCH 1/5] [RFC][AMDGPU][InsertWaitCnt] Move WaitEventType into
separate HWEvent header
I propose to move `WaitEventType` into its own header to start a new
component of the back-end targeted at analyzing and treating hardware events
fired by instructions. Right now this just moves code around and renames things
(NFCI) but over time, we should generalize the events so they can be reused
by other passes instead of being hyper-specialized for InsertWaitCnt.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 400 +++++++-----------
.../Target/AMDGPU/Utils/AMDGPUHWEvents.cpp | 34 ++
.../Target/AMDGPU/Utils/AMDGPUHWEvents.def | 65 +++
llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h | 117 +++++
llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt | 1 +
5 files changed, 360 insertions(+), 257 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 0839c6936c15e..b34a1bffcdef3 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -29,6 +29,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPUHWEvents.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Sequence.h"
@@ -44,6 +45,9 @@
using namespace llvm;
+using HWEventSet = AMDGPU::HWEventSet;
+using HWEvent = AMDGPU::HWEvent;
+
#define DEBUG_TYPE "si-insert-waitcnts"
DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE "-forceexp",
@@ -134,70 +138,10 @@ static constexpr VMEMID toVMEMID(MCRegUnit RU) {
return static_cast<unsigned>(RU);
}
-#define AMDGPU_DECLARE_WAIT_EVENTS(DECL) \
- DECL(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */ \
- DECL(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */ \
- DECL(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */ \
- DECL(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */ \
- DECL(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */ \
- DECL(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */ \
- DECL(VMEM_GROUP) /* vmem group */ \
- DECL(LDS_ACCESS) /* lds read & write */ \
- DECL(GDS_ACCESS) /* gds read & write */ \
- DECL(SQ_MESSAGE) /* send message */ \
- DECL(SCC_WRITE) /* write to SCC from barrier */ \
- DECL(SMEM_ACCESS) /* scalar-memory read & write */ \
- DECL(SMEM_GROUP) /* scalar-memory group */ \
- DECL(EXP_GPR_LOCK) /* export holding on its data src */ \
- DECL(GDS_GPR_LOCK) /* GDS holding on its data and addr src */ \
- DECL(EXP_POS_ACCESS) /* write to export position */ \
- DECL(EXP_PARAM_ACCESS) /* write to export parameter */ \
- DECL(VMW_GPR_LOCK) /* vmem write holding on its data src */ \
- DECL(EXP_LDS_ACCESS) /* read by ldsdir counting as export */ \
- DECL(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */ \
- DECL(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */ \
- DECL(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */ \
- DECL(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */ \
- DECL(VGPR_LDS_READ) /* read VGPR source in LDS */ \
- DECL(VGPR_FLAT_READ) /* read VGPR source in FLAT */ \
- DECL(VGPR_VMEM_READ) /* read VGPR source in other VMEM */ \
- DECL(ASYNC_ACCESS) /* access that uses ASYNC_CNT */ \
- DECL(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
-
-// clang-format off
-#define AMDGPU_EVENT_ENUM(Name) Name,
-enum WaitEventType {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_ENUM)
- NUM_WAIT_EVENTS
-};
-#undef AMDGPU_EVENT_ENUM
} // namespace
-namespace llvm {
-template <> struct enum_iteration_traits<WaitEventType> {
- static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
namespace {
-/// Return an iterator over all events between VMEM_ACCESS (the first event)
-/// and \c MaxEvent (exclusive, default value yields an enumeration over
-/// all counters).
-auto wait_events(WaitEventType MaxEvent = NUM_WAIT_EVENTS) {
- return enum_seq(VMEM_ACCESS, MaxEvent);
-}
-
-#define AMDGPU_EVENT_NAME(Name) #Name,
-static constexpr StringLiteral WaitEventTypeName[] = {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_NAME)
-};
-#undef AMDGPU_EVENT_NAME
-static constexpr StringLiteral getWaitEventTypeName(WaitEventType Event) {
- return WaitEventTypeName[Event];
-}
-// clang-format on
-
// Enumerate different types of result-returning VMEM operations. Although
// s_waitcnt orders them all with a single vmcnt counter, in the absence of
// s_waitcnt only instructions of the same VmemType are guaranteed to write
@@ -274,78 +218,6 @@ void addWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T, unsigned Count) {
void setNoWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) {
Wait.set(T, ~0u);
}
-
-/// A small set of events.
-class WaitEventSet {
- unsigned Mask = 0;
-
-public:
- WaitEventSet() = default;
- explicit constexpr WaitEventSet(WaitEventType Event) {
- static_assert(NUM_WAIT_EVENTS <= sizeof(Mask) * 8,
- "Not enough bits in Mask for all the events");
- Mask |= 1 << Event;
- }
- constexpr WaitEventSet(std::initializer_list<WaitEventType> Events) {
- for (auto &E : Events) {
- Mask |= 1 << E;
- }
- }
- void insert(const WaitEventType &Event) { Mask |= 1 << Event; }
- void remove(const WaitEventType &Event) { Mask &= ~(1 << Event); }
- void remove(const WaitEventSet &Other) { Mask &= ~Other.Mask; }
- bool contains(const WaitEventType &Event) const {
- return Mask & (1 << Event);
- }
- /// \Returns true if this set contains all elements of \p Other.
- bool contains(const WaitEventSet &Other) const {
- return (~Mask & Other.Mask) == 0;
- }
- /// \Returns the intersection of this and \p Other.
- WaitEventSet operator&(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask &= Other.Mask;
- return Copy;
- }
- /// \Returns the union of this and \p Other.
- WaitEventSet operator|(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask |= Other.Mask;
- return Copy;
- }
- /// This set becomes the union of this and \p Other.
- WaitEventSet &operator|=(const WaitEventSet &Other) {
- Mask |= Other.Mask;
- return *this;
- }
- /// This set becomes the intersection of this and \p Other.
- WaitEventSet &operator&=(const WaitEventSet &Other) {
- Mask &= Other.Mask;
- return *this;
- }
- bool operator==(const WaitEventSet &Other) const {
- return Mask == Other.Mask;
- }
- bool operator!=(const WaitEventSet &Other) const { return !(*this == Other); }
- bool empty() const { return Mask == 0; }
- /// \Returns true if the set contains more than one element.
- bool twoOrMore() const { return Mask & (Mask - 1); }
- operator bool() const { return !empty(); }
- void print(raw_ostream &OS) const {
- ListSeparator LS(", ");
- for (WaitEventType Event : wait_events()) {
- if (contains(Event))
- OS << LS << getWaitEventTypeName(Event);
- }
- }
- LLVM_DUMP_METHOD void dump() const;
-};
-
-void WaitEventSet::dump() const {
- print(dbgs());
- dbgs() << "\n";
-}
-
class WaitcntBrackets;
// This abstracts the logic for generating and updating S_WAIT* instructions
@@ -411,12 +283,11 @@ class WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) = 0;
- // Returns the WaitEventSet that corresponds to counter \p T.
- virtual const WaitEventSet &
- getWaitEvents(AMDGPU::InstCounterType T) const = 0;
+ // Returns the HWEventSet that corresponds to counter \p T.
+ virtual const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const = 0;
/// \returns the counter that corresponds to event \p E.
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
for (auto T : AMDGPU::inst_counter_types()) {
if (getWaitEvents(T).contains(E))
return T;
@@ -435,22 +306,25 @@ class WaitcntGenerator {
};
class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet(
- {VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet()};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet({HWEvent::SMEM_ACCESS, HWEvent::LDS_ACCESS,
+ HWEvent::GDS_ACCESS, HWEvent::SQ_MESSAGE}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet()};
public:
using WaitcntGenerator::WaitcntGenerator;
@@ -464,7 +338,7 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstPreGFX12[T];
}
@@ -474,22 +348,26 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
protected:
bool IsExpertMode;
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet({VMEM_ACCESS, GLOBAL_INV_ACCESS}),
- WaitEventSet({LDS_ACCESS, GDS_ACCESS}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet({VMEM_SAMPLER_READ_ACCESS}),
- WaitEventSet({VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, SQ_MESSAGE, SCC_WRITE}),
- WaitEventSet({VMEM_GROUP, SMEM_GROUP}),
- WaitEventSet({ASYNC_ACCESS}),
- WaitEventSet({TENSOR_ACCESS}),
- WaitEventSet({VGPR_CSMACC_WRITE, VGPR_DPMACC_WRITE, VGPR_TRANS_WRITE,
- VGPR_XDL_WRITE}),
- WaitEventSet({VGPR_LDS_READ, VGPR_FLAT_READ, VGPR_VMEM_READ})};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::GLOBAL_INV_ACCESS}),
+ HWEventSet({HWEvent::LDS_ACCESS, HWEvent::GDS_ACCESS}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet({HWEvent::VMEM_SAMPLER_READ_ACCESS}),
+ HWEventSet({HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet(
+ {HWEvent::SMEM_ACCESS, HWEvent::SQ_MESSAGE, HWEvent::SCC_WRITE}),
+ HWEventSet({HWEvent::VMEM_GROUP, HWEvent::SMEM_GROUP}),
+ HWEventSet({HWEvent::ASYNC_ACCESS}),
+ HWEventSet({HWEvent::TENSOR_ACCESS}),
+ HWEventSet({HWEvent::VGPR_CSMACC_WRITE, HWEvent::VGPR_DPMACC_WRITE,
+ HWEvent::VGPR_TRANS_WRITE, HWEvent::VGPR_XDL_WRITE}),
+ HWEventSet({HWEvent::VGPR_LDS_READ, HWEvent::VGPR_FLAT_READ,
+ HWEvent::VGPR_VMEM_READ})};
public:
WaitcntGeneratorGFX12Plus() = delete;
@@ -509,7 +387,7 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstGFX12Plus[T];
}
@@ -624,40 +502,41 @@ class SIInsertWaitcnts {
// Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM
// instruction.
- WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
+ HWEvent getVmemHWEvent(const MachineInstr &Inst) const {
switch (Inst.getOpcode()) {
// FIXME: GLOBAL_INV needs to be tracked with xcnt too.
case AMDGPU::GLOBAL_INV:
- return GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't write
- // VGPRs
+ return HWEvent::GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't
+ // write VGPRs
case AMDGPU::GLOBAL_WB:
case AMDGPU::GLOBAL_WBINV:
- return VMEM_WRITE_ACCESS; // tracked using storecnt
+ return HWEvent::VMEM_WRITE_ACCESS; // tracked using storecnt
default:
break;
}
- // Maps VMEM access types to their corresponding WaitEventType.
- static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
- VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
+ // Maps VMEM access types to their corresponding HWEvent.
+ static const HWEvent VmemReadMapping[NUM_VMEM_TYPES] = {
+ HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS};
assert(SIInstrInfo::isVMEM(Inst));
// LDS DMA loads are also stores, but on the LDS side. On the VMEM side
// these should use VM_CNT.
if (!ST.hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
if (Inst.mayStore() &&
(!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
if (TII.mayAccessScratch(Inst))
- return SCRATCH_WRITE_ACCESS;
- return VMEM_WRITE_ACCESS;
+ return HWEvent::SCRATCH_WRITE_ACCESS;
+ return HWEvent::VMEM_WRITE_ACCESS;
}
if (!ST.hasExtendedWaitCounts() || SIInstrInfo::isFLAT(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
return VmemReadMapping[getVmemType(Inst)];
}
- std::optional<WaitEventType>
+ std::optional<HWEvent>
getExpertSchedulingEventType(const MachineInstr &Inst) const;
bool isAsync(const MachineInstr &MI) const {
@@ -699,7 +578,7 @@ class SIInsertWaitcnts {
MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
MachineInstr *OldWaitcntInstr);
/// \returns all events that correspond to \p Inst.
- WaitEventSet getEventsFor(const MachineInstr &Inst) const;
+ HWEventSet getEventsFor(const MachineInstr &Inst) const;
void updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets);
bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
@@ -713,10 +592,10 @@ class SIInsertWaitcnts {
bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
bool ExpertMode) const;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
return WCG->getWaitEvents(T);
}
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
return WCG->getCounterFromEvent(E);
}
};
@@ -826,13 +705,11 @@ class WaitcntBrackets {
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
- void updateByEvent(WaitEventType E, MachineInstr &MI);
+ void updateByEvent(HWEvent E, MachineInstr &MI);
void recordAsyncMark(MachineInstr &MI);
bool hasPendingEvent() const { return !PendingEvents.empty(); }
- bool hasPendingEvent(WaitEventType E) const {
- return PendingEvents.contains(E);
- }
+ bool hasPendingEvent(HWEvent E) const { return PendingEvents.contains(E); }
bool hasPendingEvent(AMDGPU::InstCounterType T) const {
bool HasPending = PendingEvents & Context->getWaitEvents(T);
assert(HasPending == !empty(T) &&
@@ -841,7 +718,7 @@ class WaitcntBrackets {
}
bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
- WaitEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
// Return true if more than one bit is set in Events.
return Events.twoOrMore();
}
@@ -983,7 +860,7 @@ class WaitcntBrackets {
unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
- WaitEventSet PendingEvents;
+ HWEventSet PendingEvents;
// Remember the last flat memory operation.
unsigned LastFlatDsCnt = 0;
unsigned LastFlatLoadCnt = 0;
@@ -1121,7 +998,7 @@ bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
return hasOtherPendingVmemTypes(Reg, VMEM_NOSAMPLER);
}
-void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
+void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
assert(T < Context->MaxCounter);
@@ -1218,7 +1095,8 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
}
}
} else if (T == AMDGPU::X_CNT) {
- WaitEventType OtherEvent = E == SMEM_GROUP ? VMEM_GROUP : SMEM_GROUP;
+ HWEvent OtherEvent =
+ E == HWEvent::SMEM_GROUP ? HWEvent::VMEM_GROUP : HWEvent::SMEM_GROUP;
if (PendingEvents.contains(OtherEvent)) {
// Hardware inserts an implicit xcnt between interleaved
// SMEM and VMEM operations. So there will never be
@@ -1434,9 +1312,10 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
OS << "Pending Events: ";
if (hasPendingEvent()) {
ListSeparator LS;
- for (unsigned I = 0; I != NUM_WAIT_EVENTS; ++I) {
- if (hasPendingEvent((WaitEventType)I)) {
- OS << LS << WaitEventTypeName[I];
+ // FIXME: use hw_events()
+ for (auto E : AMDGPU::hw_events()) {
+ if (hasPendingEvent(E)) {
+ OS << LS << AMDGPU::toString(E);
}
}
} else {
@@ -1540,12 +1419,14 @@ void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
// Wait on XCNT is redundant if we are already waiting for a load to complete.
// SMEM can return out of order, so only omit XCNT wait if we are waiting till
// zero.
- if (CheckWait.get(AMDGPU::KM_CNT) == 0 && hasPendingEvent(SMEM_GROUP))
+ if (CheckWait.get(AMDGPU::KM_CNT) == 0 &&
+ hasPendingEvent(HWEvent::SMEM_GROUP))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
// If we have pending store we cannot optimize XCnt because we do not wait for
// stores. VMEM loads retun in order, so if we only have loads XCnt is
// decremented to the same number as LOADCnt.
- if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u && hasPendingEvent(VMEM_GROUP) &&
+ if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u &&
+ hasPendingEvent(HWEvent::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT) &&
CheckWait.get(AMDGPU::X_CNT) >= CheckWait.get(AMDGPU::LOAD_CNT))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
@@ -1678,9 +1559,9 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
return Reg32;
// If hi/lo16 mixed events
- WaitEventSet MIEvents = Context->getEventsFor(MI);
- WaitEventSet OtherHalfEvents = Context->getWaitEvents(T);
- WaitEventSet Events = MIEvents & OtherHalfEvents;
+ HWEventSet MIEvents = Context->getEventsFor(MI);
+ HWEventSet OtherHalfEvents = Context->getWaitEvents(T);
+ HWEventSet Events = MIEvents & OtherHalfEvents;
if (Events.twoOrMore())
return Reg32;
return Reg;
@@ -1716,7 +1597,7 @@ void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
if (PendingSCCWrite &&
PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
PendingSCCWrite->getOperand(0).getImm() == Inst->getOperand(0).getImm()) {
- WaitEventSet SCC_WRITE_PendingEvent(SCC_WRITE);
+ HWEventSet SCC_WRITE_PendingEvent(HWEvent::SCC_WRITE);
// If this SCC_WRITE is the only pending KM_CNT event, clear counter.
if ((PendingEvents & Context->getWaitEvents(AMDGPU::KM_CNT)) ==
SCC_WRITE_PendingEvent) {
@@ -1746,18 +1627,19 @@ void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
PendingEvents.remove(Context->getWaitEvents(T));
}
- if (T == AMDGPU::KM_CNT && Count == 0 && hasPendingEvent(SMEM_GROUP)) {
+ if (T == AMDGPU::KM_CNT && Count == 0 &&
+ hasPendingEvent(HWEvent::SMEM_GROUP)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, 0);
else
- PendingEvents.remove(SMEM_GROUP);
+ PendingEvents.remove(HWEvent::SMEM_GROUP);
}
- if (T == AMDGPU::LOAD_CNT && hasPendingEvent(VMEM_GROUP) &&
+ if (T == AMDGPU::LOAD_CNT && hasPendingEvent(HWEvent::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, Count);
else if (Count == 0)
- PendingEvents.remove(VMEM_GROUP);
+ PendingEvents.remove(HWEvent::VMEM_GROUP);
}
}
@@ -1771,18 +1653,19 @@ void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
// the decrement may go out of order.
bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
// Scalar memory read always can go out of order.
- if ((T == Context->SmemAccessCounter && hasPendingEvent(SMEM_ACCESS)) ||
- (T == AMDGPU::X_CNT && hasPendingEvent(SMEM_GROUP)))
+ if ((T == Context->SmemAccessCounter &&
+ hasPendingEvent(HWEvent::SMEM_ACCESS)) ||
+ (T == AMDGPU::X_CNT && hasPendingEvent(HWEvent::SMEM_GROUP)))
return true;
// GLOBAL_INV completes in-order with other LOAD_CNT events (VMEM_ACCESS),
// so having GLOBAL_INV_ACCESS mixed with other LOAD_CNT events doesn't cause
// out-of-order completion.
if (T == AMDGPU::LOAD_CNT) {
- WaitEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
// Remove GLOBAL_INV_ACCESS from the event mask before checking for mixed
// events
- Events.remove(GLOBAL_INV_ACCESS);
+ Events.remove(HWEvent::GLOBAL_INV_ACCESS);
// Return true only if there are still multiple event types after removing
// GLOBAL_INV
return Events.twoOrMore();
@@ -2562,7 +2445,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// GLOBAL_INV increments loadcnt but doesn't write to VGPRs, so there's
// no need to wait for it at function boundaries.
if (ST.hasExtendedWaitCounts() &&
- !ScoreBrackets.hasPendingEvent(VMEM_ACCESS))
+ !ScoreBrackets.hasPendingEvent(HWEvent::VMEM_ACCESS))
AllZeroWait.set(AMDGPU::LOAD_CNT, ~0u);
Wait = AllZeroWait;
break;
@@ -2577,8 +2460,9 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// to send a message to explicitly release all VGPRs before the stores have
// completed, but it is only safe to do this if there are no outstanding
// scratch stores.
- EndPgmInsts[&MI] = !ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
- !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS);
+ EndPgmInsts[&MI] =
+ !ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
+ !ScoreBrackets.hasPendingEvent(HWEvent::SCRATCH_WRITE_ACCESS);
break;
}
case AMDGPU::S_SENDMSG:
@@ -2601,10 +2485,10 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (MI.modifiesRegister(AMDGPU::EXEC, &TRI)) {
// Export and GDS are tracked individually, either may trigger a waitcnt
// for EXEC.
- if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
- ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
- ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
- ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
+ if (ScoreBrackets.hasPendingEvent(HWEvent::EXP_GPR_LOCK) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_PARAM_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_POS_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::GDS_GPR_LOCK)) {
Wait.set(AMDGPU::EXP_CNT, 0);
}
}
@@ -2735,7 +2619,8 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
ScoreBrackets.clearVgprVmemTypes(Reg);
}
- if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
+ if (Op.isDef() ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_LDS_ACCESS)) {
ScoreBrackets.determineWaitForPhysReg(AMDGPU::EXP_CNT, Reg, Wait,
MI);
}
@@ -2775,7 +2660,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// after fixing the scheduler. Also, the Shader Compiler code is
// independent of target.
if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
- ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
+ ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS)) {
Wait.set(AMDGPU::DS_CNT, 0);
}
@@ -2864,7 +2749,7 @@ bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
return Modified;
}
-std::optional<WaitEventType>
+std::optional<HWEvent>
SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
if (TII.isVALU(Inst)) {
// Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
@@ -2872,15 +2757,15 @@ SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
// has its own event.
if (TII.isXDL(Inst))
- return VGPR_XDL_WRITE;
+ return HWEvent::VGPR_XDL_WRITE;
if (TII.isTRANS(Inst))
- return VGPR_TRANS_WRITE;
+ return HWEvent::VGPR_TRANS_WRITE;
if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
- return VGPR_DPMACC_WRITE;
+ return HWEvent::VGPR_DPMACC_WRITE;
- return VGPR_CSMACC_WRITE;
+ return HWEvent::VGPR_CSMACC_WRITE;
}
// FLAT and LDS instructions may read their VGPR sources out-of-order
@@ -2888,13 +2773,13 @@ SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
// each of these also has a separate event.
if (TII.isFLAT(Inst))
- return VGPR_FLAT_READ;
+ return HWEvent::VGPR_FLAT_READ;
if (TII.isDS(Inst))
- return VGPR_LDS_READ;
+ return HWEvent::VGPR_LDS_READ;
if (TII.isVMEM(Inst) || TII.isVIMAGE(Inst) || TII.isVSAMPLE(Inst))
- return VGPR_VMEM_READ;
+ return HWEvent::VGPR_VMEM_READ;
// Otherwise, no hazard.
@@ -2964,8 +2849,8 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
return Result;
}
-WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
- WaitEventSet Events;
+HWEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
+ HWEventSet Events;
if (IsExpertMode) {
if (const auto ET = getExpertSchedulingEventType(Inst))
Events.insert(*ET);
@@ -2974,23 +2859,23 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
if (TII.isAlwaysGDS(Inst.getOpcode()) ||
TII.hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
- Events.insert(GDS_ACCESS);
- Events.insert(GDS_GPR_LOCK);
+ Events.insert(HWEvent::GDS_ACCESS);
+ Events.insert(HWEvent::GDS_GPR_LOCK);
} else {
- Events.insert(LDS_ACCESS);
+ Events.insert(HWEvent::LDS_ACCESS);
}
} else if (TII.isFLAT(Inst)) {
if (SIInstrInfo::isGFX12CacheInvOrWBInst(Inst.getOpcode())) {
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(getVmemHWEvent(Inst));
} else {
assert(Inst.mayLoadOrStore());
if (TII.mayAccessVMEMThroughFlat(Inst)) {
if (ST.hasWaitXcnt())
- Events.insert(VMEM_GROUP);
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(HWEvent::VMEM_GROUP);
+ Events.insert(getVmemHWEvent(Inst));
}
if (TII.mayAccessLDSThroughFlat(Inst))
- Events.insert(LDS_ACCESS);
+ Events.insert(HWEvent::LDS_ACCESS);
}
} else if (SIInstrInfo::isVMEM(Inst) &&
(!AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode()) ||
@@ -2999,41 +2884,41 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
// followed "S_WAITCNT vmcnt(0)" is needed after to ensure the writeback has
// completed.
if (ST.hasWaitXcnt())
- Events.insert(VMEM_GROUP);
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(HWEvent::VMEM_GROUP);
+ Events.insert(getVmemHWEvent(Inst));
if (ST.vmemWriteNeedsExpWaitcnt() &&
(Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst))) {
- Events.insert(VMW_GPR_LOCK);
+ Events.insert(HWEvent::VMW_GPR_LOCK);
}
} else if (TII.isSMRD(Inst)) {
if (ST.hasWaitXcnt())
- Events.insert(SMEM_GROUP);
- Events.insert(SMEM_ACCESS);
+ Events.insert(HWEvent::SMEM_GROUP);
+ Events.insert(HWEvent::SMEM_ACCESS);
} else if (SIInstrInfo::isLDSDIR(Inst)) {
- Events.insert(EXP_LDS_ACCESS);
+ Events.insert(HWEvent::EXP_LDS_ACCESS);
} else if (SIInstrInfo::isEXP(Inst)) {
unsigned Imm = TII.getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
- Events.insert(EXP_PARAM_ACCESS);
+ Events.insert(HWEvent::EXP_PARAM_ACCESS);
else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
- Events.insert(EXP_POS_ACCESS);
+ Events.insert(HWEvent::EXP_POS_ACCESS);
else
- Events.insert(EXP_GPR_LOCK);
+ Events.insert(HWEvent::EXP_GPR_LOCK);
} else if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
- Events.insert(SCC_WRITE);
+ Events.insert(HWEvent::SCC_WRITE);
} else {
switch (Inst.getOpcode()) {
case AMDGPU::S_SENDMSG:
case AMDGPU::S_SENDMSG_RTN_B32:
case AMDGPU::S_SENDMSG_RTN_B64:
case AMDGPU::S_SENDMSGHALT:
- Events.insert(SQ_MESSAGE);
+ Events.insert(HWEvent::SQ_MESSAGE);
break;
case AMDGPU::S_MEMTIME:
case AMDGPU::S_MEMREALTIME:
case AMDGPU::S_GET_BARRIER_STATE_M0:
case AMDGPU::S_GET_BARRIER_STATE_IMM:
- Events.insert(SMEM_ACCESS);
+ Events.insert(HWEvent::SMEM_ACCESS);
break;
}
}
@@ -3043,8 +2928,8 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets) {
- WaitEventSet InstEvents = getEventsFor(Inst);
- for (WaitEventType E : wait_events()) {
+ HWEventSet InstEvents = getEventsFor(Inst);
+ for (HWEvent E : AMDGPU::hw_events()) {
if (InstEvents.contains(E))
ScoreBrackets->updateByEvent(E, Inst);
}
@@ -3065,10 +2950,10 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
ScoreBrackets->setPendingFlat();
}
if (SIInstrInfo::usesASYNC_CNT(Inst)) {
- ScoreBrackets->updateByEvent(ASYNC_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvent::ASYNC_ACCESS, Inst);
}
} else if (SIInstrInfo::usesTENSOR_CNT(Inst)) {
- ScoreBrackets->updateByEvent(TENSOR_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvent::TENSOR_ACCESS, Inst);
} else if (Inst.isCall()) {
// Act as a wait on everything, but AsyncCnt and TensorCnt are never
// included in such blanket waits.
@@ -3184,9 +3069,9 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
for (auto T : inst_counter_types(Context->MaxCounter)) {
// Merge event flags for this counter
- const WaitEventSet &EventsForT = Context->getWaitEvents(T);
- const WaitEventSet OldEvents = PendingEvents & EventsForT;
- const WaitEventSet OtherEvents = Other.PendingEvents & EventsForT;
+ const HWEventSet &EventsForT = Context->getWaitEvents(T);
+ const HWEventSet OldEvents = PendingEvents & EventsForT;
+ const HWEventSet OtherEvents = Other.PendingEvents & EventsForT;
if (!OldEvents.contains(OtherEvents))
StrictDom = true;
PendingEvents |= OtherEvents;
@@ -3216,8 +3101,8 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
if (T == AMDGPU::KM_CNT) {
StrictDom |= mergeScore(M, SCCScore, Other.SCCScore);
- if (Other.hasPendingEvent(SCC_WRITE)) {
- if (!OldEvents.contains(SCC_WRITE)) {
+ if (Other.hasPendingEvent(HWEvent::SCC_WRITE)) {
+ if (!OldEvents.contains(HWEvent::SCC_WRITE)) {
PendingSCCWrite = Other.PendingSCCWrite;
} else if (PendingSCCWrite != Other.PendingSCCWrite) {
PendingSCCWrite = nullptr;
@@ -3345,7 +3230,8 @@ class VCCZWorkaround {
// If MI is a vcc write with no pending smem, or there is a pending smem
// but the target does not suffer from the vccz corruption bug, then we
// don't need to recompute vccz as this write will recompute it anyway.
- if (!ScoreBrackets.hasPendingEvent(SMEM_ACCESS) || !VCCZCorruptionBug) {
+ if (!ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS) ||
+ !VCCZCorruptionBug) {
// Compute PartiallyWritesToVCCOpt if we haven't done so already.
if (!PartiallyWritesToVCCOpt)
PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
@@ -3801,7 +3687,7 @@ bool SIInsertWaitcnts::run() {
MF, AMDGPU::NUM_NORMAL_INST_CNTS, Limits);
}
- SmemAccessCounter = getCounterFromEvent(SMEM_ACCESS);
+ SmemAccessCounter = getCounterFromEvent(HWEvent::SMEM_ACCESS);
bool Modified = false;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
new file mode 100644
index 0000000000000..f570eebe806a3
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
@@ -0,0 +1,34 @@
+//===- AMDGPUHWEvents.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUHWEvents.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace AMDGPU {
+// NOLINTNEXTLINE
+iota_range<HWEvent> hw_events(HWEvent MaxEvent) {
+ return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
+}
+
+void HWEventSet::print(raw_ostream &OS) const {
+ ListSeparator LS(", ");
+ for (HWEvent Event : hw_events()) {
+ if (contains(Event))
+ OS << LS << toString(Event);
+ }
+}
+
+void HWEventSet::dump() const {
+ print(dbgs());
+ dbgs() << "\n";
+}
+} // namespace AMDGPU
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
new file mode 100644
index 0000000000000..e0db74e93021e
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
@@ -0,0 +1,65 @@
+//===--- AMDGPUHWEvents.def -----------------------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains descriptions of the various hardware events that can
+// be tracked by the compiler.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+// clang-format off
+
+#ifndef AMDGPU_LAST_HW_EVENT
+#define AMDGPU_LAST_HW_EVENT(X)
+#endif
+
+#ifndef AMDGPU_FIRST_HW_EVENT
+#define AMDGPU_FIRST_HW_EVENT(X)
+#endif
+
+// TODO: VMEM_ACCESS should be broken up and be target-independent, not interpreted differently
+// depending on the target.
+AMDGPU_HW_EVENT(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */
+AMDGPU_HW_EVENT(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */
+AMDGPU_HW_EVENT(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */
+AMDGPU_HW_EVENT(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */
+AMDGPU_HW_EVENT(VMEM_GROUP) /* vmem group */
+AMDGPU_HW_EVENT(LDS_ACCESS) /* lds read & write */
+AMDGPU_HW_EVENT(GDS_ACCESS) /* gds read & write */
+AMDGPU_HW_EVENT(SQ_MESSAGE) /* send message */
+AMDGPU_HW_EVENT(SCC_WRITE) /* write to SCC from barrier */
+AMDGPU_HW_EVENT(SMEM_ACCESS) /* scalar-memory read & write */
+AMDGPU_HW_EVENT(SMEM_GROUP) /* scalar-memory group */
+AMDGPU_HW_EVENT(EXP_GPR_LOCK) /* export holding on its data src */
+AMDGPU_HW_EVENT(GDS_GPR_LOCK) /* GDS holding on its data and addr src */
+AMDGPU_HW_EVENT(EXP_POS_ACCESS) /* write to export position */
+AMDGPU_HW_EVENT(EXP_PARAM_ACCESS) /* write to export parameter */
+AMDGPU_HW_EVENT(VMW_GPR_LOCK) /* vmem write holding on its data src */
+AMDGPU_HW_EVENT(EXP_LDS_ACCESS) /* read by ldsdir counting as export */
+AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */
+AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */
+AMDGPU_HW_EVENT(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */
+AMDGPU_HW_EVENT(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */
+AMDGPU_HW_EVENT(VGPR_LDS_READ) /* read VGPR source in LDS */
+AMDGPU_HW_EVENT(VGPR_FLAT_READ) /* read VGPR source in FLAT */
+AMDGPU_HW_EVENT(VGPR_VMEM_READ) /* read VGPR source in other VMEM */
+AMDGPU_HW_EVENT(ASYNC_ACCESS) /* access that uses ASYNC_CNT */
+AMDGPU_HW_EVENT(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
+
+AMDGPU_FIRST_HW_EVENT(VMEM_ACCESS)
+AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
+
+
+// clang-format on
+
+#undef AMDGPU_HW_EVENT
+#undef AMDGPU_LAST_HW_EVENT
+#undef AMDGPU_FIRST_HW_EVENT
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
new file mode 100644
index 0000000000000..0c46a4b2b01c5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
@@ -0,0 +1,117 @@
+//===- AMDGPUHWEvents.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
+#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
+
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class raw_ostream;
+
+namespace AMDGPU {
+
+enum class HWEvent : unsigned char {
+#define AMDGPU_HW_EVENT(X) X,
+#define AMDGPU_FIRST_HW_EVENT(X) FIRST_WAIT_EVENT = X,
+#define AMDGPU_LAST_HW_EVENT(X) NUM_WAIT_EVENTS = X,
+#include "AMDGPUHWEvents.def"
+};
+
+} // namespace AMDGPU
+
+template <> struct enum_iteration_traits<AMDGPU::HWEvent> {
+ static constexpr bool is_iterable = true; // NOLINT
+};
+
+namespace AMDGPU {
+
+static constexpr StringLiteral toString(HWEvent Event) {
+ switch (Event) {
+#define AMDGPU_HW_EVENT(EVENT) \
+ case HWEvent::EVENT: \
+ return #EVENT;
+#include "AMDGPUHWEvents.def"
+ }
+
+ return "";
+}
+
+/// Return an iterator over all events between FIRST_WAIT_EVENT
+/// and \c MaxEvent (exclusive, default value yields an enumeration over
+/// all counters).
+// NOLINTNEXTLINE
+iota_range<HWEvent> hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS);
+
+class HWEventSet {
+ unsigned Mask = 0;
+
+public:
+ HWEventSet() = default;
+ explicit constexpr HWEventSet(HWEvent Event) {
+ static_assert(static_cast<unsigned>(HWEvent::NUM_WAIT_EVENTS) <=
+ sizeof(Mask) * 8,
+ "Not enough bits in Mask for all the events");
+ Mask |= 1 << static_cast<unsigned>(Event);
+ }
+ constexpr HWEventSet(std::initializer_list<HWEvent> Events) {
+ for (auto &E : Events) {
+ Mask |= 1 << static_cast<unsigned>(E);
+ }
+ }
+ void insert(const HWEvent &Event) {
+ Mask |= 1 << static_cast<unsigned>(Event);
+ }
+ void remove(const HWEvent &Event) {
+ Mask &= ~(1 << static_cast<unsigned>(Event));
+ }
+ void remove(const HWEventSet &Other) { Mask &= ~Other.Mask; }
+ bool contains(const HWEvent &Event) const {
+ return Mask & (1 << static_cast<unsigned>(Event));
+ }
+ /// \returns true if this set contains all elements of \p Other.
+ bool contains(const HWEventSet &Other) const {
+ return (~Mask & Other.Mask) == 0;
+ }
+ /// \returns the intersection of this and \p Other.
+ HWEventSet operator&(const HWEventSet &Other) const {
+ auto Copy = *this;
+ Copy.Mask &= Other.Mask;
+ return Copy;
+ }
+ /// \returns the union of this and \p Other.
+ HWEventSet operator|(const HWEventSet &Other) const {
+ auto Copy = *this;
+ Copy.Mask |= Other.Mask;
+ return Copy;
+ }
+ /// This set becomes the union of this and \p Other.
+ HWEventSet &operator|=(const HWEventSet &Other) {
+ Mask |= Other.Mask;
+ return *this;
+ }
+ /// This set becomes the intersection of this and \p Other.
+ HWEventSet &operator&=(const HWEventSet &Other) {
+ Mask &= Other.Mask;
+ return *this;
+ }
+ bool operator==(const HWEventSet &Other) const { return Mask == Other.Mask; }
+ bool operator!=(const HWEventSet &Other) const { return !(*this == Other); }
+ bool empty() const { return Mask == 0; }
+ /// \returns true if the set contains more than one element.
+ bool twoOrMore() const { return Mask & (Mask - 1); }
+ operator bool() const { return !empty(); }
+ void print(raw_ostream &OS) const;
+ LLVM_DUMP_METHOD void dump() const;
+};
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 7b2200d8bc488..75e56549c68cd 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMAMDGPUUtils
AMDGPUAsmUtils.cpp
AMDGPUBaseInfo.cpp
AMDGPUDelayedMCExpr.cpp
+ AMDGPUHWEvents.cpp
AMDGPUPALMetadata.cpp
AMDKernelCodeTUtils.cpp
>From c13aa4616875f542193926a0bf306eff1e61eb66 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 10 Jun 2026 11:23:07 +0200
Subject: [PATCH 2/5] Comment
---
llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp | 5 -----
llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h | 4 +++-
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
index f570eebe806a3..dfa7006454353 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
@@ -13,11 +13,6 @@
namespace llvm {
namespace AMDGPU {
-// NOLINTNEXTLINE
-iota_range<HWEvent> hw_events(HWEvent MaxEvent) {
- return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
-}
-
void HWEventSet::print(raw_ostream &OS) const {
ListSeparator LS(", ");
for (HWEvent Event : hw_events()) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
index 0c46a4b2b01c5..79460a56205e5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
@@ -47,7 +47,9 @@ static constexpr StringLiteral toString(HWEvent Event) {
/// and \c MaxEvent (exclusive, default value yields an enumeration over
/// all counters).
// NOLINTNEXTLINE
-iota_range<HWEvent> hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS);
+inline iota_range<HWEvent> hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS) {
+ return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
+}
class HWEventSet {
unsigned Mask = 0;
>From de477b61907006b42c0e7b3aed602b7878af4456 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 10 Jun 2026 11:40:03 +0200
Subject: [PATCH 3/5] fmt
---
llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
index 79460a56205e5..9b4548f18241c 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
@@ -47,7 +47,8 @@ static constexpr StringLiteral toString(HWEvent Event) {
/// and \c MaxEvent (exclusive, default value yields an enumeration over
/// all counters).
// NOLINTNEXTLINE
-inline iota_range<HWEvent> hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS) {
+inline iota_range<HWEvent>
+hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS) {
return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
}
>From d117c148c174a3e1f18a18292f7aa6460e021fcd Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 10 Jun 2026 14:10:43 +0200
Subject: [PATCH 4/5] Move files
---
llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.cpp | 0
llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.def | 0
llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.h | 0
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 2 +-
llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt | 1 -
6 files changed, 2 insertions(+), 2 deletions(-)
rename llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.cpp (100%)
rename llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.def (100%)
rename llvm/lib/Target/AMDGPU/{Utils => }/AMDGPUHWEvents.h (100%)
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
similarity index 100%
rename from llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
rename to llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
similarity index 100%
rename from llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
rename to llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
similarity index 100%
rename from llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
rename to llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index ae8f1c0fad5ba..46edc44e2cc05 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -61,6 +61,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUGlobalISelUtils.cpp
AMDGPUHazardLatency.cpp
AMDGPUHSAMetadataStreamer.cpp
+ AMDGPUHWEvents.cpp
AMDGPUInsertDelayAlu.cpp
AMDGPUInstCombineIntrinsic.cpp
AMDGPUUniformIntrinsicCombine.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index b34a1bffcdef3..4c2844927145b 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -24,12 +24,12 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "AMDGPUHWEvents.h"
#include "AMDGPUWaitcntUtils.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
-#include "Utils/AMDGPUHWEvents.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Sequence.h"
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 75e56549c68cd..7b2200d8bc488 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -2,7 +2,6 @@ add_llvm_component_library(LLVMAMDGPUUtils
AMDGPUAsmUtils.cpp
AMDGPUBaseInfo.cpp
AMDGPUDelayedMCExpr.cpp
- AMDGPUHWEvents.cpp
AMDGPUPALMetadata.cpp
AMDKernelCodeTUtils.cpp
>From c664bcb1513af0a5d6c5a7b2f7667e50908f6543 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Thu, 11 Jun 2026 13:15:06 +0200
Subject: [PATCH 5/5] Comments
---
llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h | 2 ++
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 1 -
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
index 9b4548f18241c..81487c6b20957 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
@@ -17,6 +17,8 @@ class raw_ostream;
namespace AMDGPU {
+/// TODO: This should be a bitmask from the start instead of having this enum
+/// + \ref HWEventSet below.
enum class HWEvent : unsigned char {
#define AMDGPU_HW_EVENT(X) X,
#define AMDGPU_FIRST_HW_EVENT(X) FIRST_WAIT_EVENT = X,
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 4c2844927145b..615ced1ed1442 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1312,7 +1312,6 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
OS << "Pending Events: ";
if (hasPendingEvent()) {
ListSeparator LS;
- // FIXME: use hw_events()
for (auto E : AMDGPU::hw_events()) {
if (hasPendingEvent(E)) {
OS << LS << AMDGPU::toString(E);
More information about the llvm-commits
mailing list