[llvm] [RFC][AMDGPU][InsertWaitCnt] Move WaitEventType into separate HWEvent header (PR #202886)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 01:22:01 PDT 2026
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/202886
>From 974e282fc2490929b92ee1611d8404f2b687f3d7 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Tue, 9 Jun 2026 16:54:38 +0200
Subject: [PATCH] [RFC][AMDGPU][InsertWaitCnt] Move WaitEventType into separate
HWEvent header
I propose to move `WaitEventType` into its own header to start a new
component of the back-end targeted at analyzing and treating hardware events
fired by instructions. Right now this just moves code around and renames things
(NFCI) but over time, we should generalize the events so they can be reused
by other passes instead of being hyper-specialized for InsertWaitCnt.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 400 +++++++-----------
.../Target/AMDGPU/Utils/AMDGPUHWEvents.cpp | 34 ++
.../Target/AMDGPU/Utils/AMDGPUHWEvents.def | 65 +++
llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h | 117 +++++
llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt | 1 +
5 files changed, 360 insertions(+), 257 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7241c0db726ce..28238b604b377 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -29,6 +29,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPUHWEvents.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Sequence.h"
@@ -44,6 +45,9 @@
using namespace llvm;
+using HWEventSet = AMDGPU::HWEventSet;
+using HWEvent = AMDGPU::HWEvent;
+
#define DEBUG_TYPE "si-insert-waitcnts"
DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE "-forceexp",
@@ -134,70 +138,10 @@ static constexpr VMEMID toVMEMID(MCRegUnit RU) {
return static_cast<unsigned>(RU);
}
-#define AMDGPU_DECLARE_WAIT_EVENTS(DECL) \
- DECL(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */ \
- DECL(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */ \
- DECL(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */ \
- DECL(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */ \
- DECL(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */ \
- DECL(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */ \
- DECL(VMEM_GROUP) /* vmem group */ \
- DECL(LDS_ACCESS) /* lds read & write */ \
- DECL(GDS_ACCESS) /* gds read & write */ \
- DECL(SQ_MESSAGE) /* send message */ \
- DECL(SCC_WRITE) /* write to SCC from barrier */ \
- DECL(SMEM_ACCESS) /* scalar-memory read & write */ \
- DECL(SMEM_GROUP) /* scalar-memory group */ \
- DECL(EXP_GPR_LOCK) /* export holding on its data src */ \
- DECL(GDS_GPR_LOCK) /* GDS holding on its data and addr src */ \
- DECL(EXP_POS_ACCESS) /* write to export position */ \
- DECL(EXP_PARAM_ACCESS) /* write to export parameter */ \
- DECL(VMW_GPR_LOCK) /* vmem write holding on its data src */ \
- DECL(EXP_LDS_ACCESS) /* read by ldsdir counting as export */ \
- DECL(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */ \
- DECL(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */ \
- DECL(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */ \
- DECL(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */ \
- DECL(VGPR_LDS_READ) /* read VGPR source in LDS */ \
- DECL(VGPR_FLAT_READ) /* read VGPR source in FLAT */ \
- DECL(VGPR_VMEM_READ) /* read VGPR source in other VMEM */ \
- DECL(ASYNC_ACCESS) /* access that uses ASYNC_CNT */ \
- DECL(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
-
-// clang-format off
-#define AMDGPU_EVENT_ENUM(Name) Name,
-enum WaitEventType {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_ENUM)
- NUM_WAIT_EVENTS
-};
-#undef AMDGPU_EVENT_ENUM
} // namespace
-namespace llvm {
-template <> struct enum_iteration_traits<WaitEventType> {
- static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
namespace {
-/// Return an iterator over all events between VMEM_ACCESS (the first event)
-/// and \c MaxEvent (exclusive, default value yields an enumeration over
-/// all counters).
-auto wait_events(WaitEventType MaxEvent = NUM_WAIT_EVENTS) {
- return enum_seq(VMEM_ACCESS, MaxEvent);
-}
-
-#define AMDGPU_EVENT_NAME(Name) #Name,
-static constexpr StringLiteral WaitEventTypeName[] = {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_NAME)
-};
-#undef AMDGPU_EVENT_NAME
-static constexpr StringLiteral getWaitEventTypeName(WaitEventType Event) {
- return WaitEventTypeName[Event];
-}
-// clang-format on
-
// Enumerate different types of result-returning VMEM operations. Although
// s_waitcnt orders them all with a single vmcnt counter, in the absence of
// s_waitcnt only instructions of the same VmemType are guaranteed to write
@@ -274,78 +218,6 @@ void addWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T, unsigned Count) {
void setNoWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) {
Wait.set(T, ~0u);
}
-
-/// A small set of events.
-class WaitEventSet {
- unsigned Mask = 0;
-
-public:
- WaitEventSet() = default;
- explicit constexpr WaitEventSet(WaitEventType Event) {
- static_assert(NUM_WAIT_EVENTS <= sizeof(Mask) * 8,
- "Not enough bits in Mask for all the events");
- Mask |= 1 << Event;
- }
- constexpr WaitEventSet(std::initializer_list<WaitEventType> Events) {
- for (auto &E : Events) {
- Mask |= 1 << E;
- }
- }
- void insert(const WaitEventType &Event) { Mask |= 1 << Event; }
- void remove(const WaitEventType &Event) { Mask &= ~(1 << Event); }
- void remove(const WaitEventSet &Other) { Mask &= ~Other.Mask; }
- bool contains(const WaitEventType &Event) const {
- return Mask & (1 << Event);
- }
- /// \Returns true if this set contains all elements of \p Other.
- bool contains(const WaitEventSet &Other) const {
- return (~Mask & Other.Mask) == 0;
- }
- /// \Returns the intersection of this and \p Other.
- WaitEventSet operator&(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask &= Other.Mask;
- return Copy;
- }
- /// \Returns the union of this and \p Other.
- WaitEventSet operator|(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask |= Other.Mask;
- return Copy;
- }
- /// This set becomes the union of this and \p Other.
- WaitEventSet &operator|=(const WaitEventSet &Other) {
- Mask |= Other.Mask;
- return *this;
- }
- /// This set becomes the intersection of this and \p Other.
- WaitEventSet &operator&=(const WaitEventSet &Other) {
- Mask &= Other.Mask;
- return *this;
- }
- bool operator==(const WaitEventSet &Other) const {
- return Mask == Other.Mask;
- }
- bool operator!=(const WaitEventSet &Other) const { return !(*this == Other); }
- bool empty() const { return Mask == 0; }
- /// \Returns true if the set contains more than one element.
- bool twoOrMore() const { return Mask & (Mask - 1); }
- operator bool() const { return !empty(); }
- void print(raw_ostream &OS) const {
- ListSeparator LS(", ");
- for (WaitEventType Event : wait_events()) {
- if (contains(Event))
- OS << LS << getWaitEventTypeName(Event);
- }
- }
- LLVM_DUMP_METHOD void dump() const;
-};
-
-void WaitEventSet::dump() const {
- print(dbgs());
- dbgs() << "\n";
-}
-
class WaitcntBrackets;
// This abstracts the logic for generating and updating S_WAIT* instructions
@@ -411,12 +283,11 @@ class WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) = 0;
- // Returns the WaitEventSet that corresponds to counter \p T.
- virtual const WaitEventSet &
- getWaitEvents(AMDGPU::InstCounterType T) const = 0;
+ // Returns the HWEventSet that corresponds to counter \p T.
+ virtual const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const = 0;
/// \returns the counter that corresponds to event \p E.
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
for (auto T : AMDGPU::inst_counter_types()) {
if (getWaitEvents(T).contains(E))
return T;
@@ -435,22 +306,25 @@ class WaitcntGenerator {
};
class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet(
- {VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet()};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet({HWEvent::SMEM_ACCESS, HWEvent::LDS_ACCESS,
+ HWEvent::GDS_ACCESS, HWEvent::SQ_MESSAGE}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet()};
public:
using WaitcntGenerator::WaitcntGenerator;
@@ -464,7 +338,7 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstPreGFX12[T];
}
@@ -474,22 +348,26 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
protected:
bool IsExpertMode;
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet({VMEM_ACCESS, GLOBAL_INV_ACCESS}),
- WaitEventSet({LDS_ACCESS, GDS_ACCESS}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet({VMEM_SAMPLER_READ_ACCESS}),
- WaitEventSet({VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, SQ_MESSAGE, SCC_WRITE}),
- WaitEventSet({VMEM_GROUP, SMEM_GROUP}),
- WaitEventSet({ASYNC_ACCESS}),
- WaitEventSet({TENSOR_ACCESS}),
- WaitEventSet({VGPR_CSMACC_WRITE, VGPR_DPMACC_WRITE, VGPR_TRANS_WRITE,
- VGPR_XDL_WRITE}),
- WaitEventSet({VGPR_LDS_READ, VGPR_FLAT_READ, VGPR_VMEM_READ})};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::GLOBAL_INV_ACCESS}),
+ HWEventSet({HWEvent::LDS_ACCESS, HWEvent::GDS_ACCESS}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet({HWEvent::VMEM_SAMPLER_READ_ACCESS}),
+ HWEventSet({HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet(
+ {HWEvent::SMEM_ACCESS, HWEvent::SQ_MESSAGE, HWEvent::SCC_WRITE}),
+ HWEventSet({HWEvent::VMEM_GROUP, HWEvent::SMEM_GROUP}),
+ HWEventSet({HWEvent::ASYNC_ACCESS}),
+ HWEventSet({HWEvent::TENSOR_ACCESS}),
+ HWEventSet({HWEvent::VGPR_CSMACC_WRITE, HWEvent::VGPR_DPMACC_WRITE,
+ HWEvent::VGPR_TRANS_WRITE, HWEvent::VGPR_XDL_WRITE}),
+ HWEventSet({HWEvent::VGPR_LDS_READ, HWEvent::VGPR_FLAT_READ,
+ HWEvent::VGPR_VMEM_READ})};
public:
WaitcntGeneratorGFX12Plus() = delete;
@@ -509,7 +387,7 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstGFX12Plus[T];
}
@@ -624,40 +502,41 @@ class SIInsertWaitcnts {
// Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM
// instruction.
- WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
+ HWEvent getVmemHWEvent(const MachineInstr &Inst) const {
switch (Inst.getOpcode()) {
// FIXME: GLOBAL_INV needs to be tracked with xcnt too.
case AMDGPU::GLOBAL_INV:
- return GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't write
- // VGPRs
+ return HWEvent::GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't
+ // write VGPRs
case AMDGPU::GLOBAL_WB:
case AMDGPU::GLOBAL_WBINV:
- return VMEM_WRITE_ACCESS; // tracked using storecnt
+ return HWEvent::VMEM_WRITE_ACCESS; // tracked using storecnt
default:
break;
}
- // Maps VMEM access types to their corresponding WaitEventType.
- static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
- VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
+ // Maps VMEM access types to their corresponding HWEvent.
+ static const HWEvent VmemReadMapping[NUM_VMEM_TYPES] = {
+ HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS};
assert(SIInstrInfo::isVMEM(Inst));
// LDS DMA loads are also stores, but on the LDS side. On the VMEM side
// these should use VM_CNT.
if (!ST.hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
if (Inst.mayStore() &&
(!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
if (TII.mayAccessScratch(Inst))
- return SCRATCH_WRITE_ACCESS;
- return VMEM_WRITE_ACCESS;
+ return HWEvent::SCRATCH_WRITE_ACCESS;
+ return HWEvent::VMEM_WRITE_ACCESS;
}
if (!ST.hasExtendedWaitCounts() || SIInstrInfo::isFLAT(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
return VmemReadMapping[getVmemType(Inst)];
}
- std::optional<WaitEventType>
+ std::optional<HWEvent>
getExpertSchedulingEventType(const MachineInstr &Inst) const;
bool isAsync(const MachineInstr &MI) const {
@@ -699,7 +578,7 @@ class SIInsertWaitcnts {
MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
MachineInstr *OldWaitcntInstr);
/// \returns all events that correspond to \p Inst.
- WaitEventSet getEventsFor(const MachineInstr &Inst) const;
+ HWEventSet getEventsFor(const MachineInstr &Inst) const;
void updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets);
bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
@@ -713,10 +592,10 @@ class SIInsertWaitcnts {
bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
bool ExpertMode) const;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
return WCG->getWaitEvents(T);
}
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
return WCG->getCounterFromEvent(E);
}
};
@@ -826,13 +705,11 @@ class WaitcntBrackets {
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
- void updateByEvent(WaitEventType E, MachineInstr &MI);
+ void updateByEvent(HWEvent E, MachineInstr &MI);
void recordAsyncMark(MachineInstr &MI);
bool hasPendingEvent() const { return !PendingEvents.empty(); }
- bool hasPendingEvent(WaitEventType E) const {
- return PendingEvents.contains(E);
- }
+ bool hasPendingEvent(HWEvent E) const { return PendingEvents.contains(E); }
bool hasPendingEvent(AMDGPU::InstCounterType T) const {
bool HasPending = PendingEvents & Context->getWaitEvents(T);
assert(HasPending == !empty(T) &&
@@ -841,7 +718,7 @@ class WaitcntBrackets {
}
bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
- WaitEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
// Return true if more than one bit is set in Events.
return Events.twoOrMore();
}
@@ -983,7 +860,7 @@ class WaitcntBrackets {
unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
- WaitEventSet PendingEvents;
+ HWEventSet PendingEvents;
// Remember the last flat memory operation.
unsigned LastFlatDsCnt = 0;
unsigned LastFlatLoadCnt = 0;
@@ -1121,7 +998,7 @@ bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
return hasOtherPendingVmemTypes(Reg, VMEM_NOSAMPLER);
}
-void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
+void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
assert(T < Context->MaxCounter);
@@ -1217,7 +1094,8 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
}
}
} else if (T == AMDGPU::X_CNT) {
- WaitEventType OtherEvent = E == SMEM_GROUP ? VMEM_GROUP : SMEM_GROUP;
+ HWEvent OtherEvent =
+ E == HWEvent::SMEM_GROUP ? HWEvent::VMEM_GROUP : HWEvent::SMEM_GROUP;
if (PendingEvents.contains(OtherEvent)) {
// Hardware inserts an implicit xcnt between interleaved
// SMEM and VMEM operations. So there will never be
@@ -1433,9 +1311,10 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
OS << "Pending Events: ";
if (hasPendingEvent()) {
ListSeparator LS;
- for (unsigned I = 0; I != NUM_WAIT_EVENTS; ++I) {
- if (hasPendingEvent((WaitEventType)I)) {
- OS << LS << WaitEventTypeName[I];
+ // FIXME: use hw_events()
+ for (auto E : AMDGPU::hw_events()) {
+ if (hasPendingEvent(E)) {
+ OS << LS << AMDGPU::toString(E);
}
}
} else {
@@ -1539,12 +1418,14 @@ void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
// Wait on XCNT is redundant if we are already waiting for a load to complete.
// SMEM can return out of order, so only omit XCNT wait if we are waiting till
// zero.
- if (CheckWait.get(AMDGPU::KM_CNT) == 0 && hasPendingEvent(SMEM_GROUP))
+ if (CheckWait.get(AMDGPU::KM_CNT) == 0 &&
+ hasPendingEvent(HWEvent::SMEM_GROUP))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
// If we have pending store we cannot optimize XCnt because we do not wait for
// stores. VMEM loads retun in order, so if we only have loads XCnt is
// decremented to the same number as LOADCnt.
- if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u && hasPendingEvent(VMEM_GROUP) &&
+ if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u &&
+ hasPendingEvent(HWEvent::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT) &&
CheckWait.get(AMDGPU::X_CNT) >= CheckWait.get(AMDGPU::LOAD_CNT))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
@@ -1677,9 +1558,9 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
return Reg32;
// If hi/lo16 mixed events
- WaitEventSet MIEvents = Context->getEventsFor(MI);
- WaitEventSet OtherHalfEvents = Context->getWaitEvents(T);
- WaitEventSet Events = MIEvents & OtherHalfEvents;
+ HWEventSet MIEvents = Context->getEventsFor(MI);
+ HWEventSet OtherHalfEvents = Context->getWaitEvents(T);
+ HWEventSet Events = MIEvents & OtherHalfEvents;
if (Events.twoOrMore())
return Reg32;
return Reg;
@@ -1715,7 +1596,7 @@ void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
if (PendingSCCWrite &&
PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
PendingSCCWrite->getOperand(0).getImm() == Inst->getOperand(0).getImm()) {
- WaitEventSet SCC_WRITE_PendingEvent(SCC_WRITE);
+ HWEventSet SCC_WRITE_PendingEvent(HWEvent::SCC_WRITE);
// If this SCC_WRITE is the only pending KM_CNT event, clear counter.
if ((PendingEvents & Context->getWaitEvents(AMDGPU::KM_CNT)) ==
SCC_WRITE_PendingEvent) {
@@ -1745,18 +1626,19 @@ void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
PendingEvents.remove(Context->getWaitEvents(T));
}
- if (T == AMDGPU::KM_CNT && Count == 0 && hasPendingEvent(SMEM_GROUP)) {
+ if (T == AMDGPU::KM_CNT && Count == 0 &&
+ hasPendingEvent(HWEvent::SMEM_GROUP)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, 0);
else
- PendingEvents.remove(SMEM_GROUP);
+ PendingEvents.remove(HWEvent::SMEM_GROUP);
}
- if (T == AMDGPU::LOAD_CNT && hasPendingEvent(VMEM_GROUP) &&
+ if (T == AMDGPU::LOAD_CNT && hasPendingEvent(HWEvent::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, Count);
else if (Count == 0)
- PendingEvents.remove(VMEM_GROUP);
+ PendingEvents.remove(HWEvent::VMEM_GROUP);
}
}
@@ -1770,18 +1652,19 @@ void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
// the decrement may go out of order.
bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
// Scalar memory read always can go out of order.
- if ((T == Context->SmemAccessCounter && hasPendingEvent(SMEM_ACCESS)) ||
- (T == AMDGPU::X_CNT && hasPendingEvent(SMEM_GROUP)))
+ if ((T == Context->SmemAccessCounter &&
+ hasPendingEvent(HWEvent::SMEM_ACCESS)) ||
+ (T == AMDGPU::X_CNT && hasPendingEvent(HWEvent::SMEM_GROUP)))
return true;
// GLOBAL_INV completes in-order with other LOAD_CNT events (VMEM_ACCESS),
// so having GLOBAL_INV_ACCESS mixed with other LOAD_CNT events doesn't cause
// out-of-order completion.
if (T == AMDGPU::LOAD_CNT) {
- WaitEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
// Remove GLOBAL_INV_ACCESS from the event mask before checking for mixed
// events
- Events.remove(GLOBAL_INV_ACCESS);
+ Events.remove(HWEvent::GLOBAL_INV_ACCESS);
// Return true only if there are still multiple event types after removing
// GLOBAL_INV
return Events.twoOrMore();
@@ -2561,7 +2444,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// GLOBAL_INV increments loadcnt but doesn't write to VGPRs, so there's
// no need to wait for it at function boundaries.
if (ST.hasExtendedWaitCounts() &&
- !ScoreBrackets.hasPendingEvent(VMEM_ACCESS))
+ !ScoreBrackets.hasPendingEvent(HWEvent::VMEM_ACCESS))
AllZeroWait.set(AMDGPU::LOAD_CNT, ~0u);
Wait = AllZeroWait;
break;
@@ -2576,8 +2459,9 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// to send a message to explicitly release all VGPRs before the stores have
// completed, but it is only safe to do this if there are no outstanding
// scratch stores.
- EndPgmInsts[&MI] = !ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
- !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS);
+ EndPgmInsts[&MI] =
+ !ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
+ !ScoreBrackets.hasPendingEvent(HWEvent::SCRATCH_WRITE_ACCESS);
break;
}
case AMDGPU::S_SENDMSG:
@@ -2600,10 +2484,10 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (MI.modifiesRegister(AMDGPU::EXEC, &TRI)) {
// Export and GDS are tracked individually, either may trigger a waitcnt
// for EXEC.
- if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
- ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
- ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
- ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
+ if (ScoreBrackets.hasPendingEvent(HWEvent::EXP_GPR_LOCK) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_PARAM_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_POS_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvent::GDS_GPR_LOCK)) {
Wait.set(AMDGPU::EXP_CNT, 0);
}
}
@@ -2734,7 +2618,8 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
ScoreBrackets.clearVgprVmemTypes(Reg);
}
- if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
+ if (Op.isDef() ||
+ ScoreBrackets.hasPendingEvent(HWEvent::EXP_LDS_ACCESS)) {
ScoreBrackets.determineWaitForPhysReg(AMDGPU::EXP_CNT, Reg, Wait,
MI);
}
@@ -2774,7 +2659,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// after fixing the scheduler. Also, the Shader Compiler code is
// independent of target.
if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
- ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
+ ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS)) {
Wait.set(AMDGPU::DS_CNT, 0);
}
@@ -2863,7 +2748,7 @@ bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
return Modified;
}
-std::optional<WaitEventType>
+std::optional<HWEvent>
SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
if (TII.isVALU(Inst)) {
// Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
@@ -2871,15 +2756,15 @@ SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
// has its own event.
if (TII.isXDL(Inst))
- return VGPR_XDL_WRITE;
+ return HWEvent::VGPR_XDL_WRITE;
if (TII.isTRANS(Inst))
- return VGPR_TRANS_WRITE;
+ return HWEvent::VGPR_TRANS_WRITE;
if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
- return VGPR_DPMACC_WRITE;
+ return HWEvent::VGPR_DPMACC_WRITE;
- return VGPR_CSMACC_WRITE;
+ return HWEvent::VGPR_CSMACC_WRITE;
}
// FLAT and LDS instructions may read their VGPR sources out-of-order
@@ -2887,13 +2772,13 @@ SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
// each of these also has a separate event.
if (TII.isFLAT(Inst))
- return VGPR_FLAT_READ;
+ return HWEvent::VGPR_FLAT_READ;
if (TII.isDS(Inst))
- return VGPR_LDS_READ;
+ return HWEvent::VGPR_LDS_READ;
if (TII.isVMEM(Inst) || TII.isVIMAGE(Inst) || TII.isVSAMPLE(Inst))
- return VGPR_VMEM_READ;
+ return HWEvent::VGPR_VMEM_READ;
// Otherwise, no hazard.
@@ -2963,8 +2848,8 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
return Result;
}
-WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
- WaitEventSet Events;
+HWEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
+ HWEventSet Events;
if (IsExpertMode) {
if (const auto ET = getExpertSchedulingEventType(Inst))
Events.insert(*ET);
@@ -2973,23 +2858,23 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
if (TII.isAlwaysGDS(Inst.getOpcode()) ||
TII.hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
- Events.insert(GDS_ACCESS);
- Events.insert(GDS_GPR_LOCK);
+ Events.insert(HWEvent::GDS_ACCESS);
+ Events.insert(HWEvent::GDS_GPR_LOCK);
} else {
- Events.insert(LDS_ACCESS);
+ Events.insert(HWEvent::LDS_ACCESS);
}
} else if (TII.isFLAT(Inst)) {
if (SIInstrInfo::isGFX12CacheInvOrWBInst(Inst.getOpcode())) {
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(getVmemHWEvent(Inst));
} else {
assert(Inst.mayLoadOrStore());
if (TII.mayAccessVMEMThroughFlat(Inst)) {
if (ST.hasWaitXcnt())
- Events.insert(VMEM_GROUP);
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(HWEvent::VMEM_GROUP);
+ Events.insert(getVmemHWEvent(Inst));
}
if (TII.mayAccessLDSThroughFlat(Inst))
- Events.insert(LDS_ACCESS);
+ Events.insert(HWEvent::LDS_ACCESS);
}
} else if (SIInstrInfo::isVMEM(Inst) &&
(!AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode()) ||
@@ -2998,41 +2883,41 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
// followed "S_WAITCNT vmcnt(0)" is needed after to ensure the writeback has
// completed.
if (ST.hasWaitXcnt())
- Events.insert(VMEM_GROUP);
- Events.insert(getVmemWaitEventType(Inst));
+ Events.insert(HWEvent::VMEM_GROUP);
+ Events.insert(getVmemHWEvent(Inst));
if (ST.vmemWriteNeedsExpWaitcnt() &&
(Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst))) {
- Events.insert(VMW_GPR_LOCK);
+ Events.insert(HWEvent::VMW_GPR_LOCK);
}
} else if (TII.isSMRD(Inst)) {
if (ST.hasWaitXcnt())
- Events.insert(SMEM_GROUP);
- Events.insert(SMEM_ACCESS);
+ Events.insert(HWEvent::SMEM_GROUP);
+ Events.insert(HWEvent::SMEM_ACCESS);
} else if (SIInstrInfo::isLDSDIR(Inst)) {
- Events.insert(EXP_LDS_ACCESS);
+ Events.insert(HWEvent::EXP_LDS_ACCESS);
} else if (SIInstrInfo::isEXP(Inst)) {
unsigned Imm = TII.getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
- Events.insert(EXP_PARAM_ACCESS);
+ Events.insert(HWEvent::EXP_PARAM_ACCESS);
else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
- Events.insert(EXP_POS_ACCESS);
+ Events.insert(HWEvent::EXP_POS_ACCESS);
else
- Events.insert(EXP_GPR_LOCK);
+ Events.insert(HWEvent::EXP_GPR_LOCK);
} else if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
- Events.insert(SCC_WRITE);
+ Events.insert(HWEvent::SCC_WRITE);
} else {
switch (Inst.getOpcode()) {
case AMDGPU::S_SENDMSG:
case AMDGPU::S_SENDMSG_RTN_B32:
case AMDGPU::S_SENDMSG_RTN_B64:
case AMDGPU::S_SENDMSGHALT:
- Events.insert(SQ_MESSAGE);
+ Events.insert(HWEvent::SQ_MESSAGE);
break;
case AMDGPU::S_MEMTIME:
case AMDGPU::S_MEMREALTIME:
case AMDGPU::S_GET_BARRIER_STATE_M0:
case AMDGPU::S_GET_BARRIER_STATE_IMM:
- Events.insert(SMEM_ACCESS);
+ Events.insert(HWEvent::SMEM_ACCESS);
break;
}
}
@@ -3042,8 +2927,8 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets) {
- WaitEventSet InstEvents = getEventsFor(Inst);
- for (WaitEventType E : wait_events()) {
+ HWEventSet InstEvents = getEventsFor(Inst);
+ for (HWEvent E : AMDGPU::hw_events()) {
if (InstEvents.contains(E))
ScoreBrackets->updateByEvent(E, Inst);
}
@@ -3064,10 +2949,10 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
ScoreBrackets->setPendingFlat();
}
if (SIInstrInfo::usesASYNC_CNT(Inst)) {
- ScoreBrackets->updateByEvent(ASYNC_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvent::ASYNC_ACCESS, Inst);
}
} else if (SIInstrInfo::usesTENSOR_CNT(Inst)) {
- ScoreBrackets->updateByEvent(TENSOR_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvent::TENSOR_ACCESS, Inst);
} else if (Inst.isCall()) {
// Act as a wait on everything, but AsyncCnt and TensorCnt are never
// included in such blanket waits.
@@ -3183,9 +3068,9 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
for (auto T : inst_counter_types(Context->MaxCounter)) {
// Merge event flags for this counter
- const WaitEventSet &EventsForT = Context->getWaitEvents(T);
- const WaitEventSet OldEvents = PendingEvents & EventsForT;
- const WaitEventSet OtherEvents = Other.PendingEvents & EventsForT;
+ const HWEventSet &EventsForT = Context->getWaitEvents(T);
+ const HWEventSet OldEvents = PendingEvents & EventsForT;
+ const HWEventSet OtherEvents = Other.PendingEvents & EventsForT;
if (!OldEvents.contains(OtherEvents))
StrictDom = true;
PendingEvents |= OtherEvents;
@@ -3215,8 +3100,8 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
if (T == AMDGPU::KM_CNT) {
StrictDom |= mergeScore(M, SCCScore, Other.SCCScore);
- if (Other.hasPendingEvent(SCC_WRITE)) {
- if (!OldEvents.contains(SCC_WRITE)) {
+ if (Other.hasPendingEvent(HWEvent::SCC_WRITE)) {
+ if (!OldEvents.contains(HWEvent::SCC_WRITE)) {
PendingSCCWrite = Other.PendingSCCWrite;
} else if (PendingSCCWrite != Other.PendingSCCWrite) {
PendingSCCWrite = nullptr;
@@ -3344,7 +3229,8 @@ class VCCZWorkaround {
// If MI is a vcc write with no pending smem, or there is a pending smem
// but the target does not suffer from the vccz corruption bug, then we
// don't need to recompute vccz as this write will recompute it anyway.
- if (!ScoreBrackets.hasPendingEvent(SMEM_ACCESS) || !VCCZCorruptionBug) {
+ if (!ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS) ||
+ !VCCZCorruptionBug) {
// Compute PartiallyWritesToVCCOpt if we haven't done so already.
if (!PartiallyWritesToVCCOpt)
PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
@@ -3800,7 +3686,7 @@ bool SIInsertWaitcnts::run() {
MF, AMDGPU::NUM_NORMAL_INST_CNTS, Limits);
}
- SmemAccessCounter = getCounterFromEvent(SMEM_ACCESS);
+ SmemAccessCounter = getCounterFromEvent(HWEvent::SMEM_ACCESS);
bool Modified = false;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
new file mode 100644
index 0000000000000..f570eebe806a3
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp
@@ -0,0 +1,34 @@
+//===- AMDGPUHWEvents.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUHWEvents.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace AMDGPU {
+// NOLINTNEXTLINE
+iota_range<HWEvent> hw_events(HWEvent MaxEvent) {
+ return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
+}
+
+void HWEventSet::print(raw_ostream &OS) const {
+ ListSeparator LS(", ");
+ for (HWEvent Event : hw_events()) {
+ if (contains(Event))
+ OS << LS << toString(Event);
+ }
+}
+
+void HWEventSet::dump() const {
+ print(dbgs());
+ dbgs() << "\n";
+}
+} // namespace AMDGPU
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
new file mode 100644
index 0000000000000..e0db74e93021e
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def
@@ -0,0 +1,65 @@
+//===--- AMDGPUHWEvents.def -----------------------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains descriptions of the various hardware events that can
+// be tracked by the compiler.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+// clang-format off
+
+#ifndef AMDGPU_LAST_HW_EVENT
+#define AMDGPU_LAST_HW_EVENT(X)
+#endif
+
+#ifndef AMDGPU_FIRST_HW_EVENT
+#define AMDGPU_FIRST_HW_EVENT(X)
+#endif
+
+// TODO: VMEM_ACCESS should be broken up and be target-independent, not interpreted differently
+// depending on the target.
+AMDGPU_HW_EVENT(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */
+AMDGPU_HW_EVENT(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */
+AMDGPU_HW_EVENT(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */
+AMDGPU_HW_EVENT(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */
+AMDGPU_HW_EVENT(VMEM_GROUP) /* vmem group */
+AMDGPU_HW_EVENT(LDS_ACCESS) /* lds read & write */
+AMDGPU_HW_EVENT(GDS_ACCESS) /* gds read & write */
+AMDGPU_HW_EVENT(SQ_MESSAGE) /* send message */
+AMDGPU_HW_EVENT(SCC_WRITE) /* write to SCC from barrier */
+AMDGPU_HW_EVENT(SMEM_ACCESS) /* scalar-memory read & write */
+AMDGPU_HW_EVENT(SMEM_GROUP) /* scalar-memory group */
+AMDGPU_HW_EVENT(EXP_GPR_LOCK) /* export holding on its data src */
+AMDGPU_HW_EVENT(GDS_GPR_LOCK) /* GDS holding on its data and addr src */
+AMDGPU_HW_EVENT(EXP_POS_ACCESS) /* write to export position */
+AMDGPU_HW_EVENT(EXP_PARAM_ACCESS) /* write to export parameter */
+AMDGPU_HW_EVENT(VMW_GPR_LOCK) /* vmem write holding on its data src */
+AMDGPU_HW_EVENT(EXP_LDS_ACCESS) /* read by ldsdir counting as export */
+AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */
+AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */
+AMDGPU_HW_EVENT(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */
+AMDGPU_HW_EVENT(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */
+AMDGPU_HW_EVENT(VGPR_LDS_READ) /* read VGPR source in LDS */
+AMDGPU_HW_EVENT(VGPR_FLAT_READ) /* read VGPR source in FLAT */
+AMDGPU_HW_EVENT(VGPR_VMEM_READ) /* read VGPR source in other VMEM */
+AMDGPU_HW_EVENT(ASYNC_ACCESS) /* access that uses ASYNC_CNT */
+AMDGPU_HW_EVENT(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
+
+AMDGPU_FIRST_HW_EVENT(VMEM_ACCESS)
+AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
+
+
+// clang-format on
+
+#undef AMDGPU_HW_EVENT
+#undef AMDGPU_LAST_HW_EVENT
+#undef AMDGPU_FIRST_HW_EVENT
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
new file mode 100644
index 0000000000000..0c46a4b2b01c5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h
@@ -0,0 +1,117 @@
+//===- AMDGPUHWEvents.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
+#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
+
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class raw_ostream;
+
+namespace AMDGPU {
+
+enum class HWEvent : unsigned char {
+#define AMDGPU_HW_EVENT(X) X,
+#define AMDGPU_FIRST_HW_EVENT(X) FIRST_WAIT_EVENT = X,
+#define AMDGPU_LAST_HW_EVENT(X) NUM_WAIT_EVENTS = X,
+#include "AMDGPUHWEvents.def"
+};
+
+} // namespace AMDGPU
+
+template <> struct enum_iteration_traits<AMDGPU::HWEvent> {
+ static constexpr bool is_iterable = true; // NOLINT
+};
+
+namespace AMDGPU {
+
+static constexpr StringLiteral toString(HWEvent Event) {
+ switch (Event) {
+#define AMDGPU_HW_EVENT(EVENT) \
+ case HWEvent::EVENT: \
+ return #EVENT;
+#include "AMDGPUHWEvents.def"
+ }
+
+ return "";
+}
+
+/// Return an iterator over all events between FIRST_WAIT_EVENT
+/// and \c MaxEvent (exclusive, default value yields an enumeration over
+/// all counters).
+// NOLINTNEXTLINE
+iota_range<HWEvent> hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS);
+
+class HWEventSet {
+ unsigned Mask = 0;
+
+public:
+ HWEventSet() = default;
+ explicit constexpr HWEventSet(HWEvent Event) {
+ static_assert(static_cast<unsigned>(HWEvent::NUM_WAIT_EVENTS) <=
+ sizeof(Mask) * 8,
+ "Not enough bits in Mask for all the events");
+ Mask |= 1 << static_cast<unsigned>(Event);
+ }
+ constexpr HWEventSet(std::initializer_list<HWEvent> Events) {
+ for (auto &E : Events) {
+ Mask |= 1 << static_cast<unsigned>(E);
+ }
+ }
+ void insert(const HWEvent &Event) {
+ Mask |= 1 << static_cast<unsigned>(Event);
+ }
+ void remove(const HWEvent &Event) {
+ Mask &= ~(1 << static_cast<unsigned>(Event));
+ }
+ void remove(const HWEventSet &Other) { Mask &= ~Other.Mask; }
+ bool contains(const HWEvent &Event) const {
+ return Mask & (1 << static_cast<unsigned>(Event));
+ }
+ /// \returns true if this set contains all elements of \p Other.
+ bool contains(const HWEventSet &Other) const {
+ return (~Mask & Other.Mask) == 0;
+ }
+ /// \returns the intersection of this and \p Other.
+ HWEventSet operator&(const HWEventSet &Other) const {
+ auto Copy = *this;
+ Copy.Mask &= Other.Mask;
+ return Copy;
+ }
+ /// \returns the union of this and \p Other.
+ HWEventSet operator|(const HWEventSet &Other) const {
+ auto Copy = *this;
+ Copy.Mask |= Other.Mask;
+ return Copy;
+ }
+ /// This set becomes the union of this and \p Other.
+ HWEventSet &operator|=(const HWEventSet &Other) {
+ Mask |= Other.Mask;
+ return *this;
+ }
+ /// This set becomes the intersection of this and \p Other.
+ HWEventSet &operator&=(const HWEventSet &Other) {
+ Mask &= Other.Mask;
+ return *this;
+ }
+ bool operator==(const HWEventSet &Other) const { return Mask == Other.Mask; }
+ bool operator!=(const HWEventSet &Other) const { return !(*this == Other); }
+ bool empty() const { return Mask == 0; }
+ /// \returns true if the set contains more than one element.
+ bool twoOrMore() const { return Mask & (Mask - 1); }
+ operator bool() const { return !empty(); }
+ void print(raw_ostream &OS) const;
+ LLVM_DUMP_METHOD void dump() const;
+};
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 7b2200d8bc488..75e56549c68cd 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMAMDGPUUtils
AMDGPUAsmUtils.cpp
AMDGPUBaseInfo.cpp
AMDGPUDelayedMCExpr.cpp
+ AMDGPUHWEvents.cpp
AMDGPUPALMetadata.cpp
AMDKernelCodeTUtils.cpp
More information about the llvm-commits
mailing list