[llvm] 4bf16dc - [AMDGPU][InsertWaitCnts] Make HWEvent a BitMask (#203864)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 01:05:59 PDT 2026
Author: Pierre van Houtryve
Date: 2026-06-26T10:05:55+02:00
New Revision: 4bf16dccc6305237ac5bbe0e70065c2df70edf2b
URL: https://github.com/llvm/llvm-project/commit/4bf16dccc6305237ac5bbe0e70065c2df70edf2b
DIFF: https://github.com/llvm/llvm-project/commit/4bf16dccc6305237ac5bbe0e70065c2df70edf2b.diff
LOG: [AMDGPU][InsertWaitCnts] Make HWEvent a BitMask (#203864)
Follow up from comments on
https://github.com/llvm/llvm-project/pull/202886
Make HWEvent a bitmask by default instead of having both the enum, and a
separate HWEventSet. This has the advantage of streamlining the code a
bit and opening the possibility of adding "modifiers" to events, e.g. I
imagine we could now fold "VMemType" into the Events.
We already do this with things like SMEM_GROUP. At least now it's baked
into the design.
I opted for a bit more verbosity by taking inspiration from
FastMathFlags (FMF): instead of exposing a raw enum, I wrap it in a
class w/ helper function. The downside is having to reimplement all the
little bitwise ops, but the result is a cleaner, simpler interface than
a raw enum (class) w/ many helper functions. I initially tried that but
I recoiled at the sight of things like `contains(A, B)` which isn't very
clear, while `A.contains(B)` is self explanatory.
Considering HWEvent is a bitmask, I also implemented a simple iterator
to iterate over all set bits of the mask, which is a useful thing to
have as some APIs in InsertWaitCnt rely on treating one event at a time.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
index 2b58595f0b18f..ee1d507a08c9b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
@@ -15,36 +15,28 @@
namespace llvm {
namespace AMDGPU {
-void HWEventSet::print(raw_ostream &OS) const {
- ListSeparator LS(", ");
- for (HWEvent Event : hw_events()) {
- if (contains(Event))
- OS << LS << toString(Event);
- }
-}
-void HWEventSet::dump() const {
- print(dbgs());
- dbgs() << "\n";
-}
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void HWEvents::dump() const { dbgs() << *this << "\n"; }
+#endif
-static HWEventSet getExpertSchedulingEventType(const MachineInstr &Inst,
- const SIInstrInfo &TII) {
+static HWEvents getExpertSchedulingEventType(const MachineInstr &Inst,
+ const SIInstrInfo &TII) {
if (TII.isVALU(Inst, /*AllowLDSDMA=*/true) && !SIInstrInfo::isLDSDMA(Inst)) {
// Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
// out-of-order with respect to each other, so each of these classes
// has its own event.
if (TII.isXDL(Inst))
- return HWEvent::VGPR_XDL_WRITE;
+ return HWEvents::VGPR_XDL_WRITE;
if (TII.isTRANS(Inst))
- return HWEvent::VGPR_TRANS_WRITE;
+ return HWEvents::VGPR_TRANS_WRITE;
if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
- return HWEvent::VGPR_DPMACC_WRITE;
+ return HWEvents::VGPR_DPMACC_WRITE;
- return HWEvent::VGPR_CSMACC_WRITE;
+ return HWEvents::VGPR_CSMACC_WRITE;
}
// FLAT and LDS instructions may read their VGPR sources out-of-order
@@ -52,28 +44,28 @@ static HWEventSet getExpertSchedulingEventType(const MachineInstr &Inst,
// each of these also has a separate event.
if (TII.isFLAT(Inst))
- return HWEvent::VGPR_FLAT_READ;
+ return HWEvents::VGPR_FLAT_READ;
if (TII.isDS(Inst))
- return HWEvent::VGPR_LDS_READ;
+ return HWEvents::VGPR_LDS_READ;
if (TII.isVMEM(Inst) || TII.isVIMAGE(Inst) || TII.isVSAMPLE(Inst))
- return HWEvent::VGPR_VMEM_READ;
+ return HWEvents::VGPR_VMEM_READ;
// Otherwise, no hazard.
- return {};
+ return HWEvents::NONE;
}
-static HWEvent getVmemHWEvent(const MachineInstr &Inst, const GCNSubtarget &ST,
- const SIInstrInfo &TII) {
+static HWEvents getVmemHWEvent(const MachineInstr &Inst, const GCNSubtarget &ST,
+ const SIInstrInfo &TII) {
switch (Inst.getOpcode()) {
// FIXME: GLOBAL_INV needs to be tracked with xcnt too.
case AMDGPU::GLOBAL_INV:
- return HWEvent::GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't
- // write VGPRs
+ return HWEvents::GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't
+ // write VGPRs
case AMDGPU::GLOBAL_WB:
case AMDGPU::GLOBAL_WBINV:
- return HWEvent::VMEM_WRITE_ACCESS; // tracked using storecnt
+ return HWEvents::VMEM_WRITE_ACCESS; // tracked using storecnt
default:
break;
}
@@ -82,15 +74,15 @@ static HWEvent getVmemHWEvent(const MachineInstr &Inst, const GCNSubtarget &ST,
// LDS DMA loads are also stores, but on the LDS side. On the VMEM side
// these should use VM_CNT.
if (!ST.hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
- return HWEvent::VMEM_ACCESS;
+ return HWEvents::VMEM_ACCESS;
if (Inst.mayStore() &&
(!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
if (TII.mayAccessScratch(Inst))
- return HWEvent::SCRATCH_WRITE_ACCESS;
- return HWEvent::VMEM_WRITE_ACCESS;
+ return HWEvents::SCRATCH_WRITE_ACCESS;
+ return HWEvents::VMEM_WRITE_ACCESS;
}
if (!ST.hasExtendedWaitCounts() || SIInstrInfo::isFLAT(Inst))
- return HWEvent::VMEM_ACCESS;
+ return HWEvents::VMEM_ACCESS;
if (SIInstrInfo::isImage(Inst)) {
const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode());
@@ -98,27 +90,27 @@ static HWEvent getVmemHWEvent(const MachineInstr &Inst, const GCNSubtarget &ST,
AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
if (BaseInfo->BVH)
- return HWEvent::VMEM_BVH_READ_ACCESS;
+ return HWEvents::VMEM_BVH_READ_ACCESS;
// We have to make an additional check for isVSAMPLE here since some
// instructions don't have a sampler, but are still classified as sampler
// instructions for the purposes of e.g. waitcnt.
if (BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(Inst))
- return HWEvent::VMEM_SAMPLER_READ_ACCESS;
+ return HWEvents::VMEM_SAMPLER_READ_ACCESS;
}
- return HWEvent::VMEM_ACCESS;
+ return HWEvents::VMEM_ACCESS;
}
-static HWEventSet getEventsForImpl(const MachineInstr &Inst,
- const GCNSubtarget &ST,
- const SIInstrInfo &TII) {
+static HWEvents getEventsForImpl(const MachineInstr &Inst,
+ const GCNSubtarget &ST,
+ const SIInstrInfo &TII) {
if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
if (TII.isAlwaysGDS(Inst.getOpcode()) ||
TII.hasModifiersSet(Inst, AMDGPU::OpName::gds))
- return {HWEvent::GDS_ACCESS, HWEvent::GDS_GPR_LOCK};
+ return HWEvents::GDS_ACCESS | HWEvents::GDS_GPR_LOCK;
- return HWEvent::LDS_ACCESS;
+ return HWEvents::LDS_ACCESS;
}
if (TII.isFLAT(Inst)) {
@@ -126,16 +118,16 @@ static HWEventSet getEventsForImpl(const MachineInstr &Inst,
return getVmemHWEvent(Inst, ST, TII);
assert(Inst.mayLoadOrStore());
- HWEventSet S;
+ HWEvents E = HWEvents::NONE;
if (TII.mayAccessVMEMThroughFlat(Inst)) {
if (ST.hasWaitXcnt())
- S.insert(HWEvent::VMEM_GROUP);
- S.insert(getVmemHWEvent(Inst, ST, TII));
+ E |= HWEvents::VMEM_GROUP;
+ E |= getVmemHWEvent(Inst, ST, TII);
}
if (TII.mayAccessLDSThroughFlat(Inst))
- S.insert(HWEvent::LDS_ACCESS);
- return S;
+ E |= HWEvents::LDS_ACCESS;
+ return E;
}
if (SIInstrInfo::isVMEM(Inst) &&
@@ -144,37 +136,37 @@ static HWEventSet getEventsForImpl(const MachineInstr &Inst,
// BUFFER_WBL2 is included here because unlike invalidates, has to be
// followed "S_WAITCNT vmcnt(0)" is needed after to ensure the writeback has
// completed.
- HWEventSet S = {getVmemHWEvent(Inst, ST, TII)};
+ HWEvents E = getVmemHWEvent(Inst, ST, TII);
if (ST.hasWaitXcnt())
- S.insert(HWEvent::VMEM_GROUP);
+ E |= HWEvents::VMEM_GROUP;
if (ST.vmemWriteNeedsExpWaitcnt() &&
(Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst)))
- S.insert(HWEvent::VMW_GPR_LOCK);
+ E |= HWEvents::VMW_GPR_LOCK;
- return S;
+ return E;
}
if (TII.isSMRD(Inst)) {
if (ST.hasWaitXcnt())
- return {HWEvent::SMEM_GROUP, HWEvent::SMEM_ACCESS};
- return HWEvent::SMEM_ACCESS;
+ return HWEvents::SMEM_GROUP | HWEvents::SMEM_ACCESS;
+ return HWEvents::SMEM_ACCESS;
}
if (SIInstrInfo::isLDSDIR(Inst)) {
- return HWEvent::EXP_LDS_ACCESS;
+ return HWEvents::EXP_LDS_ACCESS;
}
if (SIInstrInfo::isEXP(Inst)) {
unsigned Imm = TII.getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
- return HWEvent::EXP_PARAM_ACCESS;
+ return HWEvents::EXP_PARAM_ACCESS;
if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
- return HWEvent::EXP_POS_ACCESS;
- return HWEvent::EXP_GPR_LOCK;
+ return HWEvents::EXP_POS_ACCESS;
+ return HWEvents::EXP_GPR_LOCK;
}
if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
- return HWEvent::SCC_WRITE;
+ return HWEvents::SCC_WRITE;
}
switch (Inst.getOpcode()) {
@@ -182,19 +174,19 @@ static HWEventSet getEventsForImpl(const MachineInstr &Inst,
case AMDGPU::S_SENDMSG_RTN_B32:
case AMDGPU::S_SENDMSG_RTN_B64:
case AMDGPU::S_SENDMSGHALT:
- return HWEvent::SQ_MESSAGE;
+ return HWEvents::SQ_MESSAGE;
case AMDGPU::S_MEMTIME:
case AMDGPU::S_MEMREALTIME:
case AMDGPU::S_GET_BARRIER_STATE_M0:
case AMDGPU::S_GET_BARRIER_STATE_IMM:
- return HWEvent::SMEM_ACCESS;
+ return HWEvents::SMEM_ACCESS;
}
- return {};
+ return HWEvents::NONE;
}
-HWEventSet getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
- bool IsExpertMode) {
+HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
+ bool IsExpertMode) {
const SIInstrInfo &TII = *ST.getInstrInfo();
if (IsExpertMode)
@@ -203,4 +195,14 @@ HWEventSet getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
return getEventsForImpl(Inst, ST, TII);
}
} // namespace AMDGPU
+
+raw_ostream &operator<<(raw_ostream &OS, AMDGPU::HWEvents Events) {
+ ListSeparator LS(" | ");
+#define AMDGPU_HW_EVENT(E, V) \
+ if (Events & AMDGPU::HWEvents::E) \
+ OS << LS << #E << " ";
+#include "AMDGPUHWEvents.def"
+ return OS;
+}
+
} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
index e0db74e93021e..9d2d01cee6e02 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
@@ -19,42 +19,37 @@
#define AMDGPU_LAST_HW_EVENT(X)
#endif
-#ifndef AMDGPU_FIRST_HW_EVENT
-#define AMDGPU_FIRST_HW_EVENT(X)
-#endif
-
// TODO: VMEM_ACCESS should be broken up and be target-independent, not interpreted
diff erently
// depending on the target.
-AMDGPU_HW_EVENT(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */
-AMDGPU_HW_EVENT(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */
-AMDGPU_HW_EVENT(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */
-AMDGPU_HW_EVENT(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */
-AMDGPU_HW_EVENT(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */
-AMDGPU_HW_EVENT(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */
-AMDGPU_HW_EVENT(VMEM_GROUP) /* vmem group */
-AMDGPU_HW_EVENT(LDS_ACCESS) /* lds read & write */
-AMDGPU_HW_EVENT(GDS_ACCESS) /* gds read & write */
-AMDGPU_HW_EVENT(SQ_MESSAGE) /* send message */
-AMDGPU_HW_EVENT(SCC_WRITE) /* write to SCC from barrier */
-AMDGPU_HW_EVENT(SMEM_ACCESS) /* scalar-memory read & write */
-AMDGPU_HW_EVENT(SMEM_GROUP) /* scalar-memory group */
-AMDGPU_HW_EVENT(EXP_GPR_LOCK) /* export holding on its data src */
-AMDGPU_HW_EVENT(GDS_GPR_LOCK) /* GDS holding on its data and addr src */
-AMDGPU_HW_EVENT(EXP_POS_ACCESS) /* write to export position */
-AMDGPU_HW_EVENT(EXP_PARAM_ACCESS) /* write to export parameter */
-AMDGPU_HW_EVENT(VMW_GPR_LOCK) /* vmem write holding on its data src */
-AMDGPU_HW_EVENT(EXP_LDS_ACCESS) /* read by ldsdir counting as export */
-AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */
-AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */
-AMDGPU_HW_EVENT(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */
-AMDGPU_HW_EVENT(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */
-AMDGPU_HW_EVENT(VGPR_LDS_READ) /* read VGPR source in LDS */
-AMDGPU_HW_EVENT(VGPR_FLAT_READ) /* read VGPR source in FLAT */
-AMDGPU_HW_EVENT(VGPR_VMEM_READ) /* read VGPR source in other VMEM */
-AMDGPU_HW_EVENT(ASYNC_ACCESS) /* access that uses ASYNC_CNT */
-AMDGPU_HW_EVENT(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
+AMDGPU_HW_EVENT(VMEM_ACCESS, 0) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */
+AMDGPU_HW_EVENT(VMEM_SAMPLER_READ_ACCESS, 1) /* vmem SAMPLER read (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_BVH_READ_ACCESS, 2) /* vmem BVH read (gfx12+ only) */
+AMDGPU_HW_EVENT(GLOBAL_INV_ACCESS, 3) /* GLOBAL_INV (gfx12+ only) */
+AMDGPU_HW_EVENT(VMEM_WRITE_ACCESS, 4) /* vmem write that is not scratch */
+AMDGPU_HW_EVENT(SCRATCH_WRITE_ACCESS, 5) /* vmwrite that may be scratch */
+AMDGPU_HW_EVENT(VMEM_GROUP, 6) /* vmem group */
+AMDGPU_HW_EVENT(LDS_ACCESS, 7) /* lds read & write */
+AMDGPU_HW_EVENT(GDS_ACCESS, 8) /* gds read & write */
+AMDGPU_HW_EVENT(SQ_MESSAGE, 9) /* send message */
+AMDGPU_HW_EVENT(SCC_WRITE, 10) /* write to SCC from barrier */
+AMDGPU_HW_EVENT(SMEM_ACCESS, 11) /* scalar-memory read & write */
+AMDGPU_HW_EVENT(SMEM_GROUP, 12) /* scalar-memory group */
+AMDGPU_HW_EVENT(EXP_GPR_LOCK, 13) /* export holding on its data src */
+AMDGPU_HW_EVENT(GDS_GPR_LOCK, 14) /* GDS holding on its data and addr src */
+AMDGPU_HW_EVENT(EXP_POS_ACCESS, 15) /* write to export position */
+AMDGPU_HW_EVENT(EXP_PARAM_ACCESS, 16) /* write to export parameter */
+AMDGPU_HW_EVENT(VMW_GPR_LOCK, 17) /* vmem write holding on its data src */
+AMDGPU_HW_EVENT(EXP_LDS_ACCESS, 18) /* read by ldsdir counting as export */
+AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE, 19) /* write VGPR dest in Core/Side-MACC VALU */
+AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE, 20) /* write VGPR dest in DPMACC VALU */
+AMDGPU_HW_EVENT(VGPR_TRANS_WRITE, 21) /* write VGPR dest in TRANS VALU */
+AMDGPU_HW_EVENT(VGPR_XDL_WRITE, 22) /* write VGPR dest in XDL VALU */
+AMDGPU_HW_EVENT(VGPR_LDS_READ, 23) /* read VGPR source in LDS */
+AMDGPU_HW_EVENT(VGPR_FLAT_READ, 24) /* read VGPR source in FLAT */
+AMDGPU_HW_EVENT(VGPR_VMEM_READ, 25) /* read VGPR source in other VMEM */
+AMDGPU_HW_EVENT(ASYNC_ACCESS, 26) /* access that uses ASYNC_CNT */
+AMDGPU_HW_EVENT(TENSOR_ACCESS, 27) /* access that uses TENSOR_CNT */
-AMDGPU_FIRST_HW_EVENT(VMEM_ACCESS)
AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
@@ -62,4 +57,3 @@ AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
#undef AMDGPU_HW_EVENT
#undef AMDGPU_LAST_HW_EVENT
-#undef AMDGPU_FIRST_HW_EVENT
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
index 5fad398fa56b3..5901ee40177b3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
@@ -9,8 +9,13 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUHWEVENTS_H
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MathExtras.h"
+#include <cassert>
+#include <cstdint>
+#include <iterator>
namespace llvm {
class GCNSubtarget;
@@ -19,110 +24,153 @@ class raw_ostream;
namespace AMDGPU {
-/// TODO: This should be a bitmask from the start instead of having this enum
-/// + \ref HWEventSet below.
-enum class HWEvent : unsigned char {
-#define AMDGPU_HW_EVENT(X) X,
-#define AMDGPU_FIRST_HW_EVENT(X) FIRST_WAIT_EVENT = X,
-#define AMDGPU_LAST_HW_EVENT(X) NUM_WAIT_EVENTS = X,
+/// Bit mask of hardware events.
+///
+/// This is useful to manipulate events as hardware events rarely come alone.
+/// This class implements all the usual operators one would need to manipulate a
+/// bit mask, and also supports printing to a \ref raw_ostream and iterating
+/// over all the set bits of the event mask.
+///
+/// This class behaves like a constexpr set of flags. None of the methods should
+/// be able to mutate the data unless they are assignment operators. Examples:
+/// \verbatim
+/// A |= B; // Add flags (union).
+/// A -= B; // Remove flags (substraction).
+/// A &= B; // Intersection.
+/// A ^= B; // Bitwise XOR.
+/// (bool)A; // Check whether any bits are set; A.any() also works.
+/// !A; // Check if no bits are set; A.none() also works.
+/// A.size(); // Check how many bits are set.
+/// \endverbatim
+///
+/// This type also provides certain stronger guarantees than a simple integer:
+/// - Default constructor initializes the mask to zero.
+/// - Constructor ensures undefined bits cannot be set.
+class HWEvents {
+public:
+ using value_type = uint32_t;
+
+ enum : value_type {
+ NONE = 0,
+#define AMDGPU_HW_EVENT(X, V) X = (1 << V),
+#define AMDGPU_LAST_HW_EVENT(X) HWEVENT_LAST_EVENT = X,
#include "AMDGPUHWEvents.def"
-};
-} // namespace AMDGPU
+ ALL = ((HWEVENT_LAST_EVENT << 1) - 1)
+ };
+
+ /// Iterates over the set bits of an HWEvent.
+ /// NOLINTNEXTLINE
+ class const_iterator
+ : public iterator_facade_base<const_iterator, std::forward_iterator_tag,
+ HWEvents> {
+ // The "end" iterator is also the default-constructed iterator.
+ // We naturally move towards the "end" by clearing the set bits from least
+ // to most significant.
+ HWEvents::value_type Cur = 0;
+
+ public:
+ const_iterator() = default;
+ const_iterator(HWEvents H) : Cur(H.value()) {}
+
+ bool operator==(const const_iterator &Other) const {
+ return Cur == Other.Cur;
+ }
-template <> struct enum_iteration_traits<AMDGPU::HWEvent> {
- static constexpr bool is_iterable = true; // NOLINT
-};
+ HWEvents operator*() const {
+ // Return only rightmost (least significant) bit set.
+ return Cur ? (Cur & (1 << countr_zero(Cur))) : 0;
+ }
-namespace AMDGPU {
+ const_iterator &operator++() {
+ // Keep all bits except the least significant bit set.
+ Cur &= maskTrailingZeros<HWEvents::value_type>(countr_zero(Cur) + 1);
+ return *this;
+ }
+ };
-static constexpr StringLiteral toString(HWEvent Event) {
- switch (Event) {
-#define AMDGPU_HW_EVENT(EVENT) \
- case HWEvent::EVENT: \
- return #EVENT;
-#include "AMDGPUHWEvents.def"
+ constexpr HWEvents() = default;
+ constexpr HWEvents(value_type V) : Data(V) {
+ assert((V & ALL) == V && "Bits set out of bounds!");
}
- return "";
-}
-
-/// Return an iterator over all events between FIRST_WAIT_EVENT
-/// and \c MaxEvent (exclusive, default value yields an enumeration over
-/// all counters).
-// NOLINTNEXTLINE
-inline iota_range<HWEvent>
-hw_events(HWEvent MaxEvent = HWEvent::NUM_WAIT_EVENTS) {
- return enum_seq(HWEvent::FIRST_WAIT_EVENT, MaxEvent);
-}
+ constexpr unsigned size() const { return popcount(Data); }
+ constexpr bool any() const { return Data != 0; }
+ constexpr bool none() const { return Data == 0; }
+ constexpr value_type value() const { return Data; }
-class HWEventSet {
- unsigned Mask = 0;
+ explicit constexpr operator bool() const { return any(); }
-public:
- HWEventSet() = default;
- constexpr HWEventSet(HWEvent Event) {
- static_assert(static_cast<unsigned>(HWEvent::NUM_WAIT_EVENTS) <=
- sizeof(Mask) * 8,
- "Not enough bits in Mask for all the events");
- Mask |= 1 << static_cast<unsigned>(Event);
+ constexpr bool contains(HWEvents Other) const {
+ return (~Data & Other.Data) == 0;
}
- constexpr HWEventSet(std::initializer_list<HWEvent> Events) {
- for (auto &E : Events) {
- Mask |= 1 << static_cast<unsigned>(E);
- }
+
+ const_iterator begin() const { return *this; }
+ const_iterator end() const { return {}; }
+
+ constexpr HWEvents operator|(HWEvents Other) const {
+ return Data | Other.Data;
}
- void insert(const HWEvent &Event) {
- Mask |= 1 << static_cast<unsigned>(Event);
+ constexpr HWEvents operator&(HWEvents Other) const {
+ return Data & Other.Data;
}
- void remove(const HWEvent &Event) {
- Mask &= ~(1 << static_cast<unsigned>(Event));
+ constexpr HWEvents operator^(HWEvents Other) const {
+ return Data ^ Other.Data;
}
- void remove(const HWEventSet &Other) { Mask &= ~Other.Mask; }
- bool contains(const HWEvent &Event) const {
- return Mask & (1 << static_cast<unsigned>(Event));
+
+ constexpr HWEvents operator-(HWEvents Other) const {
+ return Data & ~Other.Data;
}
- /// \returns true if this set contains all elements of \p Other.
- bool contains(const HWEventSet &Other) const {
- return (~Mask & Other.Mask) == 0;
+
+ constexpr HWEvents operator~() const { return Data ^ ALL; }
+
+ constexpr bool operator==(HWEvents Other) const { return Data == Other.Data; }
+ constexpr bool operator!=(HWEvents Other) const { return Data != Other.Data; }
+
+ constexpr HWEvents &operator|=(HWEvents Other) {
+ Data |= Other.Data;
+ return *this;
}
- /// \returns the intersection of this and \p Other.
- HWEventSet operator&(const HWEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask &= Other.Mask;
- return Copy;
+ constexpr HWEvents &operator&=(HWEvents Other) {
+ Data &= Other.Data;
+ return *this;
}
- /// \returns the union of this and \p Other.
- HWEventSet operator|(const HWEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask |= Other.Mask;
- return Copy;
+ constexpr HWEvents &operator^=(HWEvents Other) {
+ Data ^= Other.Data;
+ return *this;
}
- /// This set becomes the union of this and \p Other.
- HWEventSet &operator|=(const HWEventSet &Other) {
- Mask |= Other.Mask;
+
+ constexpr HWEvents operator-=(HWEvents Other) {
+ Data &= ~Other.Data;
return *this;
}
- /// This set becomes the intersection of this and \p Other.
- HWEventSet &operator&=(const HWEventSet &Other) {
- Mask &= Other.Mask;
+
+ /// Overload both bitwise AND operators w/ the value_type to avoid an implicit
+ /// conversion to HWEvent in this common pattern used to clear an event bit:
+ /// `Events & ~HWEvent::EVENT_TO_CLEAR`.
+ /// If we had the implicit conversion to HWEvent, we'd assert because
+ /// `~HWEvent::EVENT_TO_CLEAR` has bits set outside of `HWEvent::ALL`.
+ constexpr HWEvents operator&(value_type Other) const { return Data & Other; }
+ constexpr HWEvents &operator&=(value_type Other) {
+ Data &= Other;
return *this;
}
- bool operator==(const HWEventSet &Other) const { return Mask == Other.Mask; }
- bool operator!=(const HWEventSet &Other) const { return !(*this == Other); }
- bool empty() const { return Mask == 0; }
- /// \returns true if the set contains more than one element.
- bool twoOrMore() const { return Mask & (Mask - 1); }
- operator bool() const { return !empty(); }
- void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void dump() const;
+#endif
+
+private:
+ value_type Data = NONE;
};
-/// \returns all HWEvents triggered by \p Inst
-HWEventSet getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
- bool IsExpertMode);
+/// \returns A bitmask of HWEvent triggered by \p Inst
+HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
+ bool IsExpertMode);
} // namespace AMDGPU
+
+raw_ostream &operator<<(raw_ostream &OS, AMDGPU::HWEvents E);
} // namespace llvm
#endif
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 78381eb337a8b..93bf63cb640ef 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -44,8 +44,7 @@
using namespace llvm;
-using HWEventSet = AMDGPU::HWEventSet;
-using HWEvent = AMDGPU::HWEvent;
+using HWEvents = AMDGPU::HWEvents;
#define DEBUG_TYPE "si-insert-waitcnts"
@@ -249,13 +248,14 @@ class WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) = 0;
- // Returns the HWEventSet that corresponds to counter \p T.
- virtual const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const = 0;
+ // Returns the set of HWEvents that corresponds to counter \p T.
+ virtual const HWEvents &getWaitEvents(AMDGPU::InstCounterType T) const = 0;
/// \returns the counter that corresponds to event \p E.
- AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
+ assert(E.size() == 1 && "Cannot handle a mask of events!");
for (auto T : AMDGPU::inst_counter_types()) {
- if (getWaitEvents(T).contains(E))
+ if (getWaitEvents(T) & E)
return T;
}
llvm_unreachable("event type has no associated counter");
@@ -272,25 +272,24 @@ class WaitcntGenerator {
};
class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
- static constexpr const HWEventSet
+ static constexpr const HWEvents
WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
- HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
- HWEvent::VMEM_BVH_READ_ACCESS}),
- HWEventSet({HWEvent::SMEM_ACCESS, HWEvent::LDS_ACCESS,
- HWEvent::GDS_ACCESS, HWEvent::SQ_MESSAGE}),
- HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
- HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
- HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
- HWEventSet(
- {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
- HWEventSet(),
- HWEventSet(),
- HWEventSet(),
- HWEventSet(),
- HWEventSet(),
- HWEventSet(),
- HWEventSet(),
- HWEventSet()};
+ HWEvents::VMEM_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
+ HWEvents::VMEM_BVH_READ_ACCESS,
+ HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
+ HWEvents::SQ_MESSAGE,
+ HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
+ HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
+ HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
+ HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE,
+ HWEvents::NONE};
public:
using WaitcntGenerator::WaitcntGenerator;
@@ -304,7 +303,7 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEvents &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstPreGFX12[T];
}
@@ -314,26 +313,26 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
protected:
bool IsExpertMode;
- static constexpr const HWEventSet
+ static constexpr const HWEvents
WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
- HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::GLOBAL_INV_ACCESS}),
- HWEventSet({HWEvent::LDS_ACCESS, HWEvent::GDS_ACCESS}),
- HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
- HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
- HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
- HWEventSet(
- {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
- HWEventSet({HWEvent::VMEM_SAMPLER_READ_ACCESS}),
- HWEventSet({HWEvent::VMEM_BVH_READ_ACCESS}),
- HWEventSet(
- {HWEvent::SMEM_ACCESS, HWEvent::SQ_MESSAGE, HWEvent::SCC_WRITE}),
- HWEventSet({HWEvent::VMEM_GROUP, HWEvent::SMEM_GROUP}),
- HWEventSet({HWEvent::ASYNC_ACCESS}),
- HWEventSet({HWEvent::TENSOR_ACCESS}),
- HWEventSet({HWEvent::VGPR_CSMACC_WRITE, HWEvent::VGPR_DPMACC_WRITE,
- HWEvent::VGPR_TRANS_WRITE, HWEvent::VGPR_XDL_WRITE}),
- HWEventSet({HWEvent::VGPR_LDS_READ, HWEvent::VGPR_FLAT_READ,
- HWEvent::VGPR_VMEM_READ})};
+ HWEvents::VMEM_ACCESS | HWEvents::GLOBAL_INV_ACCESS,
+ HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
+ HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
+ HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
+ HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
+
+ HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
+ HWEvents::VMEM_SAMPLER_READ_ACCESS,
+ HWEvents::VMEM_BVH_READ_ACCESS,
+
+ HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
+ HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
+ HWEvents::ASYNC_ACCESS,
+ HWEvents::TENSOR_ACCESS,
+ HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
+ HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
+ HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
+ HWEvents::VGPR_VMEM_READ};
public:
WaitcntGeneratorGFX12Plus() = delete;
@@ -353,7 +352,7 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEvents &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstGFX12Plus[T];
}
@@ -477,10 +476,10 @@ class SIInsertWaitcnts {
bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
bool ExpertMode) const;
- const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
+ const HWEvents &getWaitEvents(AMDGPU::InstCounterType T) const {
return WCG->getWaitEvents(T);
}
- AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
return WCG->getCounterFromEvent(E);
}
};
@@ -590,22 +589,23 @@ class WaitcntBrackets {
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
- void updateByEvent(HWEvent E, MachineInstr &MI);
+ void updateByEvent(HWEvents E, MachineInstr &MI);
void recordAsyncMark(MachineInstr &MI);
- bool hasPendingEvent() const { return !PendingEvents.empty(); }
- bool hasPendingEvent(HWEvent E) const { return PendingEvents.contains(E); }
+ HWEvents getPendingEvents() const { return PendingEvents; }
+ bool hasPendingEvent() const { return PendingEvents.any(); }
+ bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(E); }
bool hasPendingEvent(AMDGPU::InstCounterType T) const {
- bool HasPending = PendingEvents & Context->getWaitEvents(T);
+ bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
assert(HasPending == !empty(T) &&
"Expected pending events iff scoreboard is not empty");
return HasPending;
}
bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
- HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEvents Events = PendingEvents & Context->getWaitEvents(T);
// Return true if more than one bit is set in Events.
- return Events.twoOrMore();
+ return Events.size() > 1;
}
bool hasPendingFlat() const {
@@ -746,7 +746,7 @@ class WaitcntBrackets {
unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
- HWEventSet PendingEvents;
+ HWEvents PendingEvents;
// Remember the last flat memory operation.
unsigned LastFlatDsCnt = 0;
unsigned LastFlatLoadCnt = 0;
@@ -884,7 +884,8 @@ bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
return hasOtherPendingVmemTypes(Reg, VMEM_NOSAMPLER);
}
-void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
+void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
+ assert(E.size() == 1 && "Expected singular event!");
AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
assert(T < Context->MaxCounter);
@@ -902,7 +903,7 @@ void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
// PendingEvents and ScoreUB need to be update regardless if this event
// changes the score of a register or not.
// Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
- PendingEvents.insert(E);
+ PendingEvents |= E;
setScoreUB(T, CurrScore);
const SIRegisterInfo &TRI = Context->TRI;
@@ -981,15 +982,15 @@ void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
}
}
} else if (T == AMDGPU::X_CNT) {
- HWEvent OtherEvent =
- E == HWEvent::SMEM_GROUP ? HWEvent::VMEM_GROUP : HWEvent::SMEM_GROUP;
+ HWEvents OtherEvent =
+ E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
if (PendingEvents.contains(OtherEvent)) {
// Hardware inserts an implicit xcnt between interleaved
// SMEM and VMEM operations. So there will never be
// outstanding address translations for both SMEM and
// VMEM at the same time.
setScoreLB(T, getScoreUB(T) - 1);
- PendingEvents.remove(OtherEvent);
+ PendingEvents -= OtherEvent;
}
for (const MachineOperand &Op : Inst.all_uses())
setScoreByOperand(Op, T, CurrScore);
@@ -1197,12 +1198,7 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
OS << "Pending Events: ";
if (hasPendingEvent()) {
- ListSeparator LS;
- for (auto E : AMDGPU::hw_events()) {
- if (hasPendingEvent(E)) {
- OS << LS << AMDGPU::toString(E);
- }
- }
+ OS << getPendingEvents();
} else {
OS << "none";
}
@@ -1305,13 +1301,13 @@ void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
// SMEM can return out of order, so only omit XCNT wait if we are waiting till
// zero.
if (CheckWait.get(AMDGPU::KM_CNT) == 0 &&
- hasPendingEvent(HWEvent::SMEM_GROUP))
+ hasPendingEvent(HWEvents::SMEM_GROUP))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
// If we have pending store we cannot optimize XCnt because we do not wait for
// stores. VMEM loads retun in order, so if we only have loads XCnt is
// decremented to the same number as LOADCnt.
if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u &&
- hasPendingEvent(HWEvent::VMEM_GROUP) &&
+ hasPendingEvent(HWEvents::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT) &&
CheckWait.get(AMDGPU::X_CNT) >= CheckWait.get(AMDGPU::LOAD_CNT))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
@@ -1443,11 +1439,11 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
return Reg32;
// If hi/lo16 mixed events
- HWEventSet MIEvents =
+ HWEvents MIEvents =
AMDGPU::getEventsFor(MI, Context->ST, Context->IsExpertMode);
- HWEventSet OtherHalfEvents = Context->getWaitEvents(T);
- HWEventSet Events = MIEvents & OtherHalfEvents;
- if (Events.twoOrMore())
+ HWEvents OtherHalfEvents = Context->getWaitEvents(T);
+ HWEvents Events = MIEvents & OtherHalfEvents;
+ if (Events.size() > 1)
return Reg32;
return Reg;
}
@@ -1482,14 +1478,14 @@ void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
if (PendingSCCWrite &&
PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
PendingSCCWrite->getOperand(0).getImm() == Inst->getOperand(0).getImm()) {
- HWEventSet SCC_WRITE_PendingEvent(HWEvent::SCC_WRITE);
+ HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
// If this SCC_WRITE is the only pending KM_CNT event, clear counter.
if ((PendingEvents & Context->getWaitEvents(AMDGPU::KM_CNT)) ==
SCC_WRITE_PendingEvent) {
setScoreLB(AMDGPU::KM_CNT, getScoreUB(AMDGPU::KM_CNT));
}
- PendingEvents.remove(SCC_WRITE_PendingEvent);
+ PendingEvents -= SCC_WRITE_PendingEvent;
PendingSCCWrite = nullptr;
}
}
@@ -1509,22 +1505,22 @@ void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
setScoreLB(T, std::max(getScoreLB(T), UB - Count));
} else {
setScoreLB(T, UB);
- PendingEvents.remove(Context->getWaitEvents(T));
+ PendingEvents -= Context->getWaitEvents(T);
}
if (T == AMDGPU::KM_CNT && Count == 0 &&
- hasPendingEvent(HWEvent::SMEM_GROUP)) {
+ hasPendingEvent(HWEvents::SMEM_GROUP)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, 0);
else
- PendingEvents.remove(HWEvent::SMEM_GROUP);
+ PendingEvents -= HWEvents::SMEM_GROUP;
}
- if (T == AMDGPU::LOAD_CNT && hasPendingEvent(HWEvent::VMEM_GROUP) &&
+ if (T == AMDGPU::LOAD_CNT && hasPendingEvent(HWEvents::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT)) {
if (!hasMixedPendingEvents(AMDGPU::X_CNT))
applyWaitcnt(AMDGPU::X_CNT, Count);
else if (Count == 0)
- PendingEvents.remove(HWEvent::VMEM_GROUP);
+ PendingEvents -= HWEvents::VMEM_GROUP;
}
}
@@ -1539,21 +1535,21 @@ void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
// Scalar memory read always can go out of order.
if ((T == Context->SmemAccessCounter &&
- hasPendingEvent(HWEvent::SMEM_ACCESS)) ||
- (T == AMDGPU::X_CNT && hasPendingEvent(HWEvent::SMEM_GROUP)))
+ hasPendingEvent(HWEvents::SMEM_ACCESS)) ||
+ (T == AMDGPU::X_CNT && hasPendingEvent(HWEvents::SMEM_GROUP)))
return true;
// GLOBAL_INV completes in-order with other LOAD_CNT events (VMEM_ACCESS),
// so having GLOBAL_INV_ACCESS mixed with other LOAD_CNT events doesn't cause
// out-of-order completion.
if (T == AMDGPU::LOAD_CNT) {
- HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEvents Events = PendingEvents & Context->getWaitEvents(T);
// Remove GLOBAL_INV_ACCESS from the event mask before checking for mixed
// events
- Events.remove(HWEvent::GLOBAL_INV_ACCESS);
+ Events -= HWEvents::GLOBAL_INV_ACCESS;
// Return true only if there are still multiple event types after removing
// GLOBAL_INV
- return Events.twoOrMore();
+ return Events.size() > 1;
}
return hasMixedPendingEvents(T);
@@ -2283,7 +2279,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// GLOBAL_INV increments loadcnt but doesn't write to VGPRs, so there's
// no need to wait for it at function boundaries.
if (ST.hasExtendedWaitCounts() &&
- !ScoreBrackets.hasPendingEvent(HWEvent::VMEM_ACCESS))
+ !ScoreBrackets.hasPendingEvent(HWEvents::VMEM_ACCESS))
AllZeroWait.set(AMDGPU::LOAD_CNT, ~0u);
Wait = AllZeroWait;
break;
@@ -2300,7 +2296,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// scratch stores.
EndPgmInsts[&MI] =
!ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
- !ScoreBrackets.hasPendingEvent(HWEvent::SCRATCH_WRITE_ACCESS);
+ !ScoreBrackets.hasPendingEvent(HWEvents::SCRATCH_WRITE_ACCESS);
break;
}
case AMDGPU::S_SENDMSG:
@@ -2323,10 +2319,10 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (MI.modifiesRegister(AMDGPU::EXEC, &TRI)) {
// Export and GDS are tracked individually, either may trigger a waitcnt
// for EXEC.
- if (ScoreBrackets.hasPendingEvent(HWEvent::EXP_GPR_LOCK) ||
- ScoreBrackets.hasPendingEvent(HWEvent::EXP_PARAM_ACCESS) ||
- ScoreBrackets.hasPendingEvent(HWEvent::EXP_POS_ACCESS) ||
- ScoreBrackets.hasPendingEvent(HWEvent::GDS_GPR_LOCK)) {
+ if (ScoreBrackets.hasPendingEvent(HWEvents::EXP_GPR_LOCK) ||
+ ScoreBrackets.hasPendingEvent(HWEvents::EXP_PARAM_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvents::EXP_POS_ACCESS) ||
+ ScoreBrackets.hasPendingEvent(HWEvents::GDS_GPR_LOCK)) {
Wait.set(AMDGPU::EXP_CNT, 0);
}
}
@@ -2458,7 +2454,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
}
if (Op.isDef() ||
- ScoreBrackets.hasPendingEvent(HWEvent::EXP_LDS_ACCESS)) {
+ ScoreBrackets.hasPendingEvent(HWEvents::EXP_LDS_ACCESS)) {
ScoreBrackets.determineWaitForPhysReg(AMDGPU::EXP_CNT, Reg, Wait,
MI);
}
@@ -2498,7 +2494,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// after fixing the scheduler. Also, the Shader Compiler code is
// independent of target.
if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
- ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS)) {
+ ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS)) {
Wait.set(AMDGPU::DS_CNT, 0);
}
@@ -2653,11 +2649,9 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets) {
- HWEventSet InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode);
- for (HWEvent E : AMDGPU::hw_events()) {
- if (InstEvents.contains(E))
- ScoreBrackets->updateByEvent(E, Inst);
- }
+ HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode);
+ for (HWEvents E : InstEvents)
+ ScoreBrackets->updateByEvent(E, Inst);
if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
if (TII.isAlwaysGDS(Inst.getOpcode()) ||
@@ -2675,10 +2669,10 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
ScoreBrackets->setPendingFlat();
}
if (SIInstrInfo::usesASYNC_CNT(Inst)) {
- ScoreBrackets->updateByEvent(HWEvent::ASYNC_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvents::ASYNC_ACCESS, Inst);
}
} else if (SIInstrInfo::usesTENSOR_CNT(Inst)) {
- ScoreBrackets->updateByEvent(HWEvent::TENSOR_ACCESS, Inst);
+ ScoreBrackets->updateByEvent(HWEvents::TENSOR_ACCESS, Inst);
} else if (Inst.isCall()) {
// Act as a wait on everything, but AsyncCnt and TensorCnt are never
// included in such blanket waits.
@@ -2794,9 +2788,9 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
for (auto T : inst_counter_types(Context->MaxCounter)) {
// Merge event flags for this counter
- const HWEventSet &EventsForT = Context->getWaitEvents(T);
- const HWEventSet OldEvents = PendingEvents & EventsForT;
- const HWEventSet OtherEvents = Other.PendingEvents & EventsForT;
+ const HWEvents &EventsForT = Context->getWaitEvents(T);
+ const HWEvents OldEvents = PendingEvents & EventsForT;
+ const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
if (!OldEvents.contains(OtherEvents))
StrictDom = true;
PendingEvents |= OtherEvents;
@@ -2826,8 +2820,8 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
if (T == AMDGPU::KM_CNT) {
StrictDom |= mergeScore(M, SCCScore, Other.SCCScore);
- if (Other.hasPendingEvent(HWEvent::SCC_WRITE)) {
- if (!OldEvents.contains(HWEvent::SCC_WRITE)) {
+ if (Other.hasPendingEvent(HWEvents::SCC_WRITE)) {
+ if (!(OldEvents & HWEvents::SCC_WRITE)) {
PendingSCCWrite = Other.PendingSCCWrite;
} else if (PendingSCCWrite != Other.PendingSCCWrite) {
PendingSCCWrite = nullptr;
@@ -2955,7 +2949,7 @@ class VCCZWorkaround {
// If MI is a vcc write with no pending smem, or there is a pending smem
// but the target does not suffer from the vccz corruption bug, then we
// don't need to recompute vccz as this write will recompute it anyway.
- if (!ScoreBrackets.hasPendingEvent(HWEvent::SMEM_ACCESS) ||
+ if (!ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS) ||
!VCCZCorruptionBug) {
// Compute PartiallyWritesToVCCOpt if we haven't done so already.
if (!PartiallyWritesToVCCOpt)
@@ -3412,7 +3406,7 @@ bool SIInsertWaitcnts::run() {
MF, AMDGPU::NUM_NORMAL_INST_CNTS, Limits);
}
- SmemAccessCounter = getCounterFromEvent(HWEvent::SMEM_ACCESS);
+ SmemAccessCounter = getCounterFromEvent(HWEvents::SMEM_ACCESS);
bool Modified = false;
More information about the llvm-commits
mailing list