[llvm] [RFC][AMDGPU][InsertWaitCnt] Move WaitEventType into separate HWEvent header (PR #202886)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 01:22:45 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Pierre van Houtryve (Pierre-vh)
<details>
<summary>Changes</summary>
I propose to move `WaitEventType` into its own header to start a new
component of the back-end targeted at analyzing and treating hardware events
fired by instructions. Right now this just moves code around and renames things
(NFCI) but over time, we should generalize the events so they can be reused
by other passes instead of being hyper-specialized for InsertWaitCnt.
---
Patch is 42.74 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/202886.diff
5 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+143-257)
- (added) llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.cpp (+34)
- (added) llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.def (+65)
- (added) llvm/lib/Target/AMDGPU/Utils/AMDGPUHWEvents.h (+117)
- (modified) llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt (+1)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7241c0db726ce..28238b604b377 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -29,6 +29,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPUHWEvents.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Sequence.h"
@@ -44,6 +45,9 @@
using namespace llvm;
+using HWEventSet = AMDGPU::HWEventSet;
+using HWEvent = AMDGPU::HWEvent;
+
#define DEBUG_TYPE "si-insert-waitcnts"
DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE "-forceexp",
@@ -134,70 +138,10 @@ static constexpr VMEMID toVMEMID(MCRegUnit RU) {
return static_cast<unsigned>(RU);
}
-#define AMDGPU_DECLARE_WAIT_EVENTS(DECL) \
- DECL(VMEM_ACCESS) /* vmem read & write (pre-gfx10), vmem read (gfx10+) */ \
- DECL(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */ \
- DECL(VMEM_BVH_READ_ACCESS) /* vmem BVH read (gfx12+ only) */ \
- DECL(GLOBAL_INV_ACCESS) /* GLOBAL_INV (gfx12+ only) */ \
- DECL(VMEM_WRITE_ACCESS) /* vmem write that is not scratch */ \
- DECL(SCRATCH_WRITE_ACCESS) /* vmem write that may be scratch */ \
- DECL(VMEM_GROUP) /* vmem group */ \
- DECL(LDS_ACCESS) /* lds read & write */ \
- DECL(GDS_ACCESS) /* gds read & write */ \
- DECL(SQ_MESSAGE) /* send message */ \
- DECL(SCC_WRITE) /* write to SCC from barrier */ \
- DECL(SMEM_ACCESS) /* scalar-memory read & write */ \
- DECL(SMEM_GROUP) /* scalar-memory group */ \
- DECL(EXP_GPR_LOCK) /* export holding on its data src */ \
- DECL(GDS_GPR_LOCK) /* GDS holding on its data and addr src */ \
- DECL(EXP_POS_ACCESS) /* write to export position */ \
- DECL(EXP_PARAM_ACCESS) /* write to export parameter */ \
- DECL(VMW_GPR_LOCK) /* vmem write holding on its data src */ \
- DECL(EXP_LDS_ACCESS) /* read by ldsdir counting as export */ \
- DECL(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */ \
- DECL(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */ \
- DECL(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */ \
- DECL(VGPR_XDL_WRITE) /* write VGPR dest in XDL VALU */ \
- DECL(VGPR_LDS_READ) /* read VGPR source in LDS */ \
- DECL(VGPR_FLAT_READ) /* read VGPR source in FLAT */ \
- DECL(VGPR_VMEM_READ) /* read VGPR source in other VMEM */ \
- DECL(ASYNC_ACCESS) /* access that uses ASYNC_CNT */ \
- DECL(TENSOR_ACCESS) /* access that uses TENSOR_CNT */
-
-// clang-format off
-#define AMDGPU_EVENT_ENUM(Name) Name,
-enum WaitEventType {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_ENUM)
- NUM_WAIT_EVENTS
-};
-#undef AMDGPU_EVENT_ENUM
} // namespace
-namespace llvm {
-template <> struct enum_iteration_traits<WaitEventType> {
- static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
namespace {
-/// Return an iterator over all events between VMEM_ACCESS (the first event)
-/// and \c MaxEvent (exclusive, default value yields an enumeration over
-/// all counters).
-auto wait_events(WaitEventType MaxEvent = NUM_WAIT_EVENTS) {
- return enum_seq(VMEM_ACCESS, MaxEvent);
-}
-
-#define AMDGPU_EVENT_NAME(Name) #Name,
-static constexpr StringLiteral WaitEventTypeName[] = {
- AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_NAME)
-};
-#undef AMDGPU_EVENT_NAME
-static constexpr StringLiteral getWaitEventTypeName(WaitEventType Event) {
- return WaitEventTypeName[Event];
-}
-// clang-format on
-
// Enumerate different types of result-returning VMEM operations. Although
// s_waitcnt orders them all with a single vmcnt counter, in the absence of
// s_waitcnt only instructions of the same VmemType are guaranteed to write
@@ -274,78 +218,6 @@ void addWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T, unsigned Count) {
void setNoWait(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) {
Wait.set(T, ~0u);
}
-
-/// A small set of events.
-class WaitEventSet {
- unsigned Mask = 0;
-
-public:
- WaitEventSet() = default;
- explicit constexpr WaitEventSet(WaitEventType Event) {
- static_assert(NUM_WAIT_EVENTS <= sizeof(Mask) * 8,
- "Not enough bits in Mask for all the events");
- Mask |= 1 << Event;
- }
- constexpr WaitEventSet(std::initializer_list<WaitEventType> Events) {
- for (auto &E : Events) {
- Mask |= 1 << E;
- }
- }
- void insert(const WaitEventType &Event) { Mask |= 1 << Event; }
- void remove(const WaitEventType &Event) { Mask &= ~(1 << Event); }
- void remove(const WaitEventSet &Other) { Mask &= ~Other.Mask; }
- bool contains(const WaitEventType &Event) const {
- return Mask & (1 << Event);
- }
- /// \Returns true if this set contains all elements of \p Other.
- bool contains(const WaitEventSet &Other) const {
- return (~Mask & Other.Mask) == 0;
- }
- /// \Returns the intersection of this and \p Other.
- WaitEventSet operator&(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask &= Other.Mask;
- return Copy;
- }
- /// \Returns the union of this and \p Other.
- WaitEventSet operator|(const WaitEventSet &Other) const {
- auto Copy = *this;
- Copy.Mask |= Other.Mask;
- return Copy;
- }
- /// This set becomes the union of this and \p Other.
- WaitEventSet &operator|=(const WaitEventSet &Other) {
- Mask |= Other.Mask;
- return *this;
- }
- /// This set becomes the intersection of this and \p Other.
- WaitEventSet &operator&=(const WaitEventSet &Other) {
- Mask &= Other.Mask;
- return *this;
- }
- bool operator==(const WaitEventSet &Other) const {
- return Mask == Other.Mask;
- }
- bool operator!=(const WaitEventSet &Other) const { return !(*this == Other); }
- bool empty() const { return Mask == 0; }
- /// \Returns true if the set contains more than one element.
- bool twoOrMore() const { return Mask & (Mask - 1); }
- operator bool() const { return !empty(); }
- void print(raw_ostream &OS) const {
- ListSeparator LS(", ");
- for (WaitEventType Event : wait_events()) {
- if (contains(Event))
- OS << LS << getWaitEventTypeName(Event);
- }
- }
- LLVM_DUMP_METHOD void dump() const;
-};
-
-void WaitEventSet::dump() const {
- print(dbgs());
- dbgs() << "\n";
-}
-
class WaitcntBrackets;
// This abstracts the logic for generating and updating S_WAIT* instructions
@@ -411,12 +283,11 @@ class WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) = 0;
- // Returns the WaitEventSet that corresponds to counter \p T.
- virtual const WaitEventSet &
- getWaitEvents(AMDGPU::InstCounterType T) const = 0;
+ // Returns the HWEventSet that corresponds to counter \p T.
+ virtual const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const = 0;
/// \returns the counter that corresponds to event \p E.
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
for (auto T : AMDGPU::inst_counter_types()) {
if (getWaitEvents(T).contains(E))
return T;
@@ -435,22 +306,25 @@ class WaitcntGenerator {
};
class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet(
- {VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet(),
- WaitEventSet()};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet({HWEvent::SMEM_ACCESS, HWEvent::LDS_ACCESS,
+ HWEvent::GDS_ACCESS, HWEvent::SQ_MESSAGE}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet(),
+ HWEventSet()};
public:
using WaitcntGenerator::WaitcntGenerator;
@@ -464,7 +338,7 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstPreGFX12[T];
}
@@ -474,22 +348,26 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
protected:
bool IsExpertMode;
- static constexpr const WaitEventSet
+ static constexpr const HWEventSet
WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
- WaitEventSet({VMEM_ACCESS, GLOBAL_INV_ACCESS}),
- WaitEventSet({LDS_ACCESS, GDS_ACCESS}),
- WaitEventSet({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK,
- EXP_PARAM_ACCESS, EXP_POS_ACCESS, EXP_LDS_ACCESS}),
- WaitEventSet({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
- WaitEventSet({VMEM_SAMPLER_READ_ACCESS}),
- WaitEventSet({VMEM_BVH_READ_ACCESS}),
- WaitEventSet({SMEM_ACCESS, SQ_MESSAGE, SCC_WRITE}),
- WaitEventSet({VMEM_GROUP, SMEM_GROUP}),
- WaitEventSet({ASYNC_ACCESS}),
- WaitEventSet({TENSOR_ACCESS}),
- WaitEventSet({VGPR_CSMACC_WRITE, VGPR_DPMACC_WRITE, VGPR_TRANS_WRITE,
- VGPR_XDL_WRITE}),
- WaitEventSet({VGPR_LDS_READ, VGPR_FLAT_READ, VGPR_VMEM_READ})};
+ HWEventSet({HWEvent::VMEM_ACCESS, HWEvent::GLOBAL_INV_ACCESS}),
+ HWEventSet({HWEvent::LDS_ACCESS, HWEvent::GDS_ACCESS}),
+ HWEventSet({HWEvent::EXP_GPR_LOCK, HWEvent::GDS_GPR_LOCK,
+ HWEvent::VMW_GPR_LOCK, HWEvent::EXP_PARAM_ACCESS,
+ HWEvent::EXP_POS_ACCESS, HWEvent::EXP_LDS_ACCESS}),
+ HWEventSet(
+ {HWEvent::VMEM_WRITE_ACCESS, HWEvent::SCRATCH_WRITE_ACCESS}),
+ HWEventSet({HWEvent::VMEM_SAMPLER_READ_ACCESS}),
+ HWEventSet({HWEvent::VMEM_BVH_READ_ACCESS}),
+ HWEventSet(
+ {HWEvent::SMEM_ACCESS, HWEvent::SQ_MESSAGE, HWEvent::SCC_WRITE}),
+ HWEventSet({HWEvent::VMEM_GROUP, HWEvent::SMEM_GROUP}),
+ HWEventSet({HWEvent::ASYNC_ACCESS}),
+ HWEventSet({HWEvent::TENSOR_ACCESS}),
+ HWEventSet({HWEvent::VGPR_CSMACC_WRITE, HWEvent::VGPR_DPMACC_WRITE,
+ HWEvent::VGPR_TRANS_WRITE, HWEvent::VGPR_XDL_WRITE}),
+ HWEventSet({HWEvent::VGPR_LDS_READ, HWEvent::VGPR_FLAT_READ,
+ HWEvent::VGPR_VMEM_READ})};
public:
WaitcntGeneratorGFX12Plus() = delete;
@@ -509,7 +387,7 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
AMDGPU::Waitcnt Wait,
const WaitcntBrackets &ScoreBrackets) override;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const override {
return WaitEventMaskForInstGFX12Plus[T];
}
@@ -624,40 +502,41 @@ class SIInsertWaitcnts {
// Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM
// instruction.
- WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
+ HWEvent getVmemHWEvent(const MachineInstr &Inst) const {
switch (Inst.getOpcode()) {
// FIXME: GLOBAL_INV needs to be tracked with xcnt too.
case AMDGPU::GLOBAL_INV:
- return GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't write
- // VGPRs
+ return HWEvent::GLOBAL_INV_ACCESS; // tracked using loadcnt, but doesn't
+ // write VGPRs
case AMDGPU::GLOBAL_WB:
case AMDGPU::GLOBAL_WBINV:
- return VMEM_WRITE_ACCESS; // tracked using storecnt
+ return HWEvent::VMEM_WRITE_ACCESS; // tracked using storecnt
default:
break;
}
- // Maps VMEM access types to their corresponding WaitEventType.
- static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
- VMEM_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
+ // Maps VMEM access types to their corresponding HWEvent.
+ static const HWEvent VmemReadMapping[NUM_VMEM_TYPES] = {
+ HWEvent::VMEM_ACCESS, HWEvent::VMEM_SAMPLER_READ_ACCESS,
+ HWEvent::VMEM_BVH_READ_ACCESS};
assert(SIInstrInfo::isVMEM(Inst));
// LDS DMA loads are also stores, but on the LDS side. On the VMEM side
// these should use VM_CNT.
if (!ST.hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
if (Inst.mayStore() &&
(!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
if (TII.mayAccessScratch(Inst))
- return SCRATCH_WRITE_ACCESS;
- return VMEM_WRITE_ACCESS;
+ return HWEvent::SCRATCH_WRITE_ACCESS;
+ return HWEvent::VMEM_WRITE_ACCESS;
}
if (!ST.hasExtendedWaitCounts() || SIInstrInfo::isFLAT(Inst))
- return VMEM_ACCESS;
+ return HWEvent::VMEM_ACCESS;
return VmemReadMapping[getVmemType(Inst)];
}
- std::optional<WaitEventType>
+ std::optional<HWEvent>
getExpertSchedulingEventType(const MachineInstr &Inst) const;
bool isAsync(const MachineInstr &MI) const {
@@ -699,7 +578,7 @@ class SIInsertWaitcnts {
MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
MachineInstr *OldWaitcntInstr);
/// \returns all events that correspond to \p Inst.
- WaitEventSet getEventsFor(const MachineInstr &Inst) const;
+ HWEventSet getEventsFor(const MachineInstr &Inst) const;
void updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets);
bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
@@ -713,10 +592,10 @@ class SIInsertWaitcnts {
bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
bool ExpertMode) const;
- const WaitEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
+ const HWEventSet &getWaitEvents(AMDGPU::InstCounterType T) const {
return WCG->getWaitEvents(T);
}
- AMDGPU::InstCounterType getCounterFromEvent(WaitEventType E) const {
+ AMDGPU::InstCounterType getCounterFromEvent(HWEvent E) const {
return WCG->getCounterFromEvent(E);
}
};
@@ -826,13 +705,11 @@ class WaitcntBrackets {
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
- void updateByEvent(WaitEventType E, MachineInstr &MI);
+ void updateByEvent(HWEvent E, MachineInstr &MI);
void recordAsyncMark(MachineInstr &MI);
bool hasPendingEvent() const { return !PendingEvents.empty(); }
- bool hasPendingEvent(WaitEventType E) const {
- return PendingEvents.contains(E);
- }
+ bool hasPendingEvent(HWEvent E) const { return PendingEvents.contains(E); }
bool hasPendingEvent(AMDGPU::InstCounterType T) const {
bool HasPending = PendingEvents & Context->getWaitEvents(T);
assert(HasPending == !empty(T) &&
@@ -841,7 +718,7 @@ class WaitcntBrackets {
}
bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
- WaitEventSet Events = PendingEvents & Context->getWaitEvents(T);
+ HWEventSet Events = PendingEvents & Context->getWaitEvents(T);
// Return true if more than one bit is set in Events.
return Events.twoOrMore();
}
@@ -983,7 +860,7 @@ class WaitcntBrackets {
unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
- WaitEventSet PendingEvents;
+ HWEventSet PendingEvents;
// Remember the last flat memory operation.
unsigned LastFlatDsCnt = 0;
unsigned LastFlatLoadCnt = 0;
@@ -1121,7 +998,7 @@ bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
return hasOtherPendingVmemTypes(Reg, VMEM_NOSAMPLER);
}
-void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
+void WaitcntBrackets::updateByEvent(HWEvent E, MachineInstr &Inst) {
AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
assert(T < Context->MaxCounter);
@@ -1217,7 +1094,8 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
}
}
} else if (T == AMDGPU::X_CNT) {
- WaitEventType OtherEvent = E == SMEM_GROUP ? VMEM_GROUP : SMEM_GROUP;
+ HWEvent OtherEvent =
+ E == HWEvent::SMEM_GROUP ? HWEvent::VMEM_GROUP : HWEvent::SMEM_GROUP;
if (PendingEvents.contains(OtherEvent)) {
// Hardware inserts an implicit xcnt between interleaved
// SMEM and VMEM operations. So there will never be
@@ -1433,9 +1311,10 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
OS << "Pending Events: ";
if (hasPendingEvent()) {
ListSeparator LS;
- for (unsigned I = 0; I != NUM_WAIT_EVENTS; ++I) {
- if (hasPendingEvent((WaitEventType)I)) {
- OS << LS << WaitEventTypeName[I];
+ // FIXME: use hw_events()
+ for (auto E : AMDGPU::hw_events()) {
+ if (hasPendingEvent(E)) {
+ OS << LS << AMDGPU::toString(E);
}
}
} else {
@@ -1539,12 +1418,14 @@ void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
// Wait on XCNT is redundant if we are already waiting for a load to complete.
// SMEM can return out of order, so only omit XCNT wait if we are waiting till
// zero.
- if (CheckWait.get(AMDGPU::KM_CNT) == 0 && hasPendingEvent(SMEM_GROUP))
+ if (CheckWait.get(AMDGPU::KM_CNT) == 0 &&
+ hasPendingEvent(HWEvent::SMEM_GROUP))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
// If we have pending store we cannot optimize XCnt because we do not wait for
// stores. VMEM loads retun in order, so if we only have loads XCnt is
// decremented to the same number as LOADCnt.
- if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u && hasPendingEvent(VMEM_GROUP) &&
+ if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u &&
+ hasPendingEvent(HWEvent::VMEM_GROUP) &&
!hasPendingEvent(AMDGPU::STORE_CNT) &&
CheckWait.get(AMDGPU::X_CNT) >= CheckWait.get(AMDGPU::LOAD_CNT))
UpdateWait.set(AMDGPU::X_CNT, ~0u);
@@ -1677,9 +1558,9 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
return Reg32;
// If hi/lo16 mixed events
- WaitEventSet MIEvents = Context->getEventsFor(MI);
- WaitEventSet OtherHalfEvents = Context->getWaitEvents(T);
- WaitEventSet Events = MIEvents & OtherHalfEvents;
+ HWEventSet MIEvents = Context->getEventsFor(MI);
+ HWEventSet OtherHalfEvents = Context->getWaitEvents(T);
+ HWEventSet Events = MIEvents & OtherHalfEvents;
if (Events.twoO...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/202886
More information about the llvm-commits
mailing list