[llvm] [AMDGPU][SIInsertWaitcnts][NFC] Move instr events code into separate function (PR #180864)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 11 08:40:59 PST 2026
https://github.com/vporpo updated https://github.com/llvm/llvm-project/pull/180864
>From 072fc1973649ed85cbc0f34b4e586d368478feac Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vasileios.porpodas at amd.com>
Date: Tue, 10 Feb 2026 23:01:11 +0000
Subject: [PATCH 1/2] [AMDGPU][SIInsertWaitcnts][NFC] Move instr events code
into separate function
This patch moves the code that finds which events correspond to an instruction
into a separate `getEventsFor(MachineInstr)` function.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 130 +++++++++-----------
1 file changed, 61 insertions(+), 69 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7dfe0da7ef81a..4f93b1ffe0738 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -650,6 +650,8 @@ class SIInsertWaitcnts {
MachineBasicBlock::instr_iterator It,
MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
MachineInstr *OldWaitcntInstr);
+ /// \returns all events that correspond to \p Inst.
+ SmallVector<WaitEventType> getEventsFor(const MachineInstr &Inst) const;
void updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets);
bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
@@ -2660,120 +2662,110 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
return Result;
}
-void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
- WaitcntBrackets *ScoreBrackets) {
- // Now look at the instruction opcode. If it is a memory access
- // instruction, update the upper-bound of the appropriate counter's
- // bracket and the destination operand scores.
- // For architectures with X_CNT, mark the source address operands
- // with the appropriate counter values.
- // TODO: Use the (TSFlags & SIInstrFlags::DS_CNT) property everywhere.
-
- bool IsVMEMAccess = false;
- bool IsSMEMAccess = false;
-
+SmallVector<WaitEventType>
+SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
+ SmallVector<WaitEventType> Events;
if (IsExpertMode) {
if (const auto ET = getExpertSchedulingEventType(Inst))
- ScoreBrackets->updateByEvent(*ET, Inst);
+ Events.push_back(*ET);
}
if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
if (TII->isAlwaysGDS(Inst.getOpcode()) ||
TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
- ScoreBrackets->updateByEvent(GDS_ACCESS, Inst);
- ScoreBrackets->updateByEvent(GDS_GPR_LOCK, Inst);
- ScoreBrackets->setPendingGDS();
+ Events.push_back(GDS_ACCESS);
+ Events.push_back(GDS_GPR_LOCK);
} else {
- ScoreBrackets->updateByEvent(LDS_ACCESS, Inst);
+ Events.push_back(LDS_ACCESS);
}
} else if (TII->isFLAT(Inst)) {
if (SIInstrInfo::isGFX12CacheInvOrWBInst(Inst.getOpcode())) {
- ScoreBrackets->updateByEvent(getVmemWaitEventType(Inst), Inst);
- return;
- }
-
- assert(Inst.mayLoadOrStore());
-
- int FlatASCount = 0;
-
- if (TII->mayAccessVMEMThroughFlat(Inst)) {
- ++FlatASCount;
- IsVMEMAccess = true;
- ScoreBrackets->updateByEvent(getVmemWaitEventType(Inst), Inst);
- }
-
- if (TII->mayAccessLDSThroughFlat(Inst)) {
- ++FlatASCount;
- ScoreBrackets->updateByEvent(LDS_ACCESS, Inst);
+ Events.push_back(getVmemWaitEventType(Inst));
+ } else {
+ assert(Inst.mayLoadOrStore());
+ if (TII->mayAccessVMEMThroughFlat(Inst)) {
+ if (ST->hasWaitXcnt())
+ Events.push_back(VMEM_GROUP);
+ Events.push_back(getVmemWaitEventType(Inst));
+ }
+ if (TII->mayAccessLDSThroughFlat(Inst))
+ Events.push_back(LDS_ACCESS);
}
-
- // Async/LDSDMA operations have FLAT encoding but do not actually use flat
- // pointers. They do have two operands that each access global and LDS, thus
- // making it appear at this point that they are using a flat pointer. Filter
- // them out, and for the rest, generate a dependency on flat pointers so
- // that both VM and LGKM counters are flushed.
- if (!SIInstrInfo::isLDSDMA(Inst) && FlatASCount > 1)
- ScoreBrackets->setPendingFlat();
} else if (SIInstrInfo::isVMEM(Inst) &&
(!AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode()) ||
Inst.getOpcode() == AMDGPU::BUFFER_WBL2)) {
// BUFFER_WBL2 is included here because unlike invalidates, has to be
// followed "S_WAITCNT vmcnt(0)" is needed after to ensure the writeback has
// completed.
- IsVMEMAccess = true;
- ScoreBrackets->updateByEvent(getVmemWaitEventType(Inst), Inst);
-
+ if (ST->hasWaitXcnt())
+ Events.push_back(VMEM_GROUP);
+ Events.push_back(getVmemWaitEventType(Inst));
if (ST->vmemWriteNeedsExpWaitcnt() &&
(Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst))) {
- ScoreBrackets->updateByEvent(VMW_GPR_LOCK, Inst);
+ Events.push_back(VMW_GPR_LOCK);
}
} else if (TII->isSMRD(Inst)) {
- IsSMEMAccess = true;
- ScoreBrackets->updateByEvent(SMEM_ACCESS, Inst);
- } else if (Inst.isCall()) {
- // Act as a wait on everything
- ScoreBrackets->applyWaitcnt(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
- ScoreBrackets->setStateOnFunctionEntryOrReturn();
+ if (ST->hasWaitXcnt())
+ Events.push_back(SMEM_GROUP);
+ Events.push_back(SMEM_ACCESS);
} else if (SIInstrInfo::isLDSDIR(Inst)) {
- ScoreBrackets->updateByEvent(EXP_LDS_ACCESS, Inst);
- } else if (TII->isVINTERP(Inst)) {
- int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
- ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
+ Events.push_back(EXP_LDS_ACCESS);
} else if (SIInstrInfo::isEXP(Inst)) {
unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
- ScoreBrackets->updateByEvent(EXP_PARAM_ACCESS, Inst);
+ Events.push_back(EXP_PARAM_ACCESS);
else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
- ScoreBrackets->updateByEvent(EXP_POS_ACCESS, Inst);
+ Events.push_back(EXP_POS_ACCESS);
else
- ScoreBrackets->updateByEvent(EXP_GPR_LOCK, Inst);
+ Events.push_back(EXP_GPR_LOCK);
} else if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
- ScoreBrackets->updateByEvent(SCC_WRITE, Inst);
+ Events.push_back(SCC_WRITE);
} else {
switch (Inst.getOpcode()) {
case AMDGPU::S_SENDMSG:
case AMDGPU::S_SENDMSG_RTN_B32:
case AMDGPU::S_SENDMSG_RTN_B64:
case AMDGPU::S_SENDMSGHALT:
- ScoreBrackets->updateByEvent(SQ_MESSAGE, Inst);
+ Events.push_back(SQ_MESSAGE);
break;
case AMDGPU::S_MEMTIME:
case AMDGPU::S_MEMREALTIME:
case AMDGPU::S_GET_BARRIER_STATE_M0:
case AMDGPU::S_GET_BARRIER_STATE_IMM:
- ScoreBrackets->updateByEvent(SMEM_ACCESS, Inst);
+ Events.push_back(SMEM_ACCESS);
break;
}
}
+ return Events;
+}
- if (!ST->hasWaitXcnt())
- return;
-
- if (IsVMEMAccess)
- ScoreBrackets->updateByEvent(VMEM_GROUP, Inst);
+void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
+ WaitcntBrackets *ScoreBrackets) {
+ for (WaitEventType E : getEventsFor(Inst))
+ ScoreBrackets->updateByEvent(E, Inst);
- if (IsSMEMAccess)
- ScoreBrackets->updateByEvent(SMEM_GROUP, Inst);
+ if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
+ if (TII->isAlwaysGDS(Inst.getOpcode()) ||
+ TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
+ ScoreBrackets->setPendingGDS();
+ }
+ } else if (TII->isFLAT(Inst)) {
+ if (Inst.mayLoadOrStore() && TII->mayAccessVMEMThroughFlat(Inst) &&
+ TII->mayAccessLDSThroughFlat(Inst) && !SIInstrInfo::isLDSDMA(Inst))
+ // Async/LDSDMA operations have FLAT encoding but do not actually use flat
+ // pointers. They do have two operands that each access global and LDS,
+ // thus making it appear at this point that they are using a flat pointer.
+ // Filter them out, and for the rest, generate a dependency on flat
+ // pointers so that both VM and LGKM counters are flushed.
+ ScoreBrackets->setPendingFlat();
+ } else if (Inst.isCall()) {
+ // Act as a wait on everything
+ ScoreBrackets->applyWaitcnt(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
+ ScoreBrackets->setStateOnFunctionEntryOrReturn();
+ } else if (TII->isVINTERP(Inst)) {
+ int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
+ ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
+ }
}
bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
>From 082958a856c22bfbd4b6f2381fb552c4a2aec094 Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vasileios.porpodas at amd.com>
Date: Wed, 11 Feb 2026 16:38:04 +0000
Subject: [PATCH 2/2] fixup! [AMDGPU][SIInsertWaitcnts][NFC] Move instr events
code into separate function
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 55 +++++++++++----------
1 file changed, 29 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 4f93b1ffe0738..d537ef31ca605 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -651,7 +651,7 @@ class SIInsertWaitcnts {
MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
MachineInstr *OldWaitcntInstr);
/// \returns all events that correspond to \p Inst.
- SmallVector<WaitEventType> getEventsFor(const MachineInstr &Inst) const;
+ WaitEventSet getEventsFor(const MachineInstr &Inst) const;
void updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets);
bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
@@ -2662,34 +2662,33 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
return Result;
}
-SmallVector<WaitEventType>
-SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
- SmallVector<WaitEventType> Events;
+WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
+ WaitEventSet Events;
if (IsExpertMode) {
if (const auto ET = getExpertSchedulingEventType(Inst))
- Events.push_back(*ET);
+ Events.insert(*ET);
}
if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
if (TII->isAlwaysGDS(Inst.getOpcode()) ||
TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
- Events.push_back(GDS_ACCESS);
- Events.push_back(GDS_GPR_LOCK);
+ Events.insert(GDS_ACCESS);
+ Events.insert(GDS_GPR_LOCK);
} else {
- Events.push_back(LDS_ACCESS);
+ Events.insert(LDS_ACCESS);
}
} else if (TII->isFLAT(Inst)) {
if (SIInstrInfo::isGFX12CacheInvOrWBInst(Inst.getOpcode())) {
- Events.push_back(getVmemWaitEventType(Inst));
+ Events.insert(getVmemWaitEventType(Inst));
} else {
assert(Inst.mayLoadOrStore());
if (TII->mayAccessVMEMThroughFlat(Inst)) {
if (ST->hasWaitXcnt())
- Events.push_back(VMEM_GROUP);
- Events.push_back(getVmemWaitEventType(Inst));
+ Events.insert(VMEM_GROUP);
+ Events.insert(getVmemWaitEventType(Inst));
}
if (TII->mayAccessLDSThroughFlat(Inst))
- Events.push_back(LDS_ACCESS);
+ Events.insert(LDS_ACCESS);
}
} else if (SIInstrInfo::isVMEM(Inst) &&
(!AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode()) ||
@@ -2698,41 +2697,41 @@ SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
// followed "S_WAITCNT vmcnt(0)" is needed after to ensure the writeback has
// completed.
if (ST->hasWaitXcnt())
- Events.push_back(VMEM_GROUP);
- Events.push_back(getVmemWaitEventType(Inst));
+ Events.insert(VMEM_GROUP);
+ Events.insert(getVmemWaitEventType(Inst));
if (ST->vmemWriteNeedsExpWaitcnt() &&
(Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst))) {
- Events.push_back(VMW_GPR_LOCK);
+ Events.insert(VMW_GPR_LOCK);
}
} else if (TII->isSMRD(Inst)) {
if (ST->hasWaitXcnt())
- Events.push_back(SMEM_GROUP);
- Events.push_back(SMEM_ACCESS);
+ Events.insert(SMEM_GROUP);
+ Events.insert(SMEM_ACCESS);
} else if (SIInstrInfo::isLDSDIR(Inst)) {
- Events.push_back(EXP_LDS_ACCESS);
+ Events.insert(EXP_LDS_ACCESS);
} else if (SIInstrInfo::isEXP(Inst)) {
unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
- Events.push_back(EXP_PARAM_ACCESS);
+ Events.insert(EXP_PARAM_ACCESS);
else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
- Events.push_back(EXP_POS_ACCESS);
+ Events.insert(EXP_POS_ACCESS);
else
- Events.push_back(EXP_GPR_LOCK);
+ Events.insert(EXP_GPR_LOCK);
} else if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
- Events.push_back(SCC_WRITE);
+ Events.insert(SCC_WRITE);
} else {
switch (Inst.getOpcode()) {
case AMDGPU::S_SENDMSG:
case AMDGPU::S_SENDMSG_RTN_B32:
case AMDGPU::S_SENDMSG_RTN_B64:
case AMDGPU::S_SENDMSGHALT:
- Events.push_back(SQ_MESSAGE);
+ Events.insert(SQ_MESSAGE);
break;
case AMDGPU::S_MEMTIME:
case AMDGPU::S_MEMREALTIME:
case AMDGPU::S_GET_BARRIER_STATE_M0:
case AMDGPU::S_GET_BARRIER_STATE_IMM:
- Events.push_back(SMEM_ACCESS);
+ Events.insert(SMEM_ACCESS);
break;
}
}
@@ -2741,8 +2740,12 @@ SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
WaitcntBrackets *ScoreBrackets) {
- for (WaitEventType E : getEventsFor(Inst))
- ScoreBrackets->updateByEvent(E, Inst);
+
+ WaitEventSet InstEvents = getEventsFor(Inst);
+ for (WaitEventType E : wait_events()) {
+ if (InstEvents.contains(E))
+ ScoreBrackets->updateByEvent(E, Inst);
+ }
if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
if (TII->isAlwaysGDS(Inst.getOpcode()) ||
More information about the llvm-commits
mailing list