[llvm] [RFC][AMDGPU][InsertWaitcnts] Move `WaitcntBrackets` to a separate file (PR #210044)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 02:20:21 PDT 2026
================
@@ -0,0 +1,1101 @@
+//===- AMDGPUWaitcntTracking.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUWaitcntTracking.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "si-insert-waitcnts"
+
+namespace llvm {
+namespace AMDGPU {
+
+namespace {
+static bool isAsync(const MachineInstr &MI, const SIInstrInfo &TII) {
+ if (!SIInstrInfo::isLDSDMA(MI))
+ return false;
+ if (SIInstrInfo::usesASYNC_CNT(MI))
+ return true;
+ const MachineOperand *Async = TII.getNamedOperand(MI, OpName::IsAsync);
+ return Async && (Async->getImm());
+}
+
+static bool isNonAsyncLdsDmaWrite(const MachineInstr &MI,
+ const SIInstrInfo &TII) {
+ return SIInstrInfo::mayWriteLDSThroughDMA(MI) && !isAsync(MI, TII);
+}
+
+static bool isAsyncLdsDmaWrite(const MachineInstr &MI, const SIInstrInfo &TII) {
+ return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI, TII);
+}
+
+static bool shouldUpdateAsyncMark(const MachineInstr &MI, InstCounterType T,
+ const SIInstrInfo &TII) {
+ if (SIInstrInfo::usesTENSOR_CNT(MI))
+ return T == TENSOR_CNT;
+ if (!isAsyncLdsDmaWrite(MI, TII))
+ return false;
+ if (SIInstrInfo::usesASYNC_CNT(MI))
+ return T == ASYNC_CNT;
+ return T == LOAD_CNT;
+}
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// WaitcntBrackets
+//===----------------------------------------------------------------------===//
+
+WaitcntBrackets::WaitcntBrackets(const WaitcntBracketsContext &Ctx)
+ : Ctx(&Ctx) {
+ assert(Ctx.TRI.getNumRegUnits() < REGUNITS_END);
+}
+
+#ifndef NDEBUG
+WaitcntBrackets::~WaitcntBrackets() {
+ unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
+ for (auto &[ID, Val] : VMem) {
+ if (Val.empty())
+ ++NumUnusedVmem;
+ }
+ for (auto &[ID, Val] : SGPRs) {
+ if (Val.empty())
+ ++NumUnusedSGPRs;
+ }
+
+ if (NumUnusedVmem || NumUnusedSGPRs) {
+ errs() << "WaitcntBracket had unused entries at destruction time: "
+ << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
+ << " SGPR unused entries\n";
+ std::abort();
+ }
+}
+#endif
+
+void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
+ InstCounterType CntTy, unsigned Score) {
+ setRegScore(Op.getReg().asMCReg(), CntTy, Score);
+}
+
+unsigned WaitcntBrackets::SGPRInfo::get(InstCounterType T) const {
+ assert((T == DS_CNT || T == KM_CNT || T == X_CNT) && "Invalid counter");
+ return T == X_CNT ? ScoreXCnt : ScoreDsKmCnt;
+}
+unsigned &WaitcntBrackets::SGPRInfo::get(InstCounterType T) {
+ assert((T == DS_CNT || T == KM_CNT || T == X_CNT) && "Invalid counter");
+ return T == X_CNT ? ScoreXCnt : ScoreDsKmCnt;
+}
+
+bool WaitcntBrackets::hasPendingFlat() const {
+ return (
+ (LastFlatDsCnt > ScoreLBs[DS_CNT] && LastFlatDsCnt <= ScoreUBs[DS_CNT]) ||
+ (LastFlatLoadCnt > ScoreLBs[LOAD_CNT] &&
+ LastFlatLoadCnt <= ScoreUBs[LOAD_CNT]));
+}
+
+void WaitcntBrackets::setPendingFlat() {
+ LastFlatLoadCnt = ScoreUBs[LOAD_CNT];
+ LastFlatDsCnt = ScoreUBs[DS_CNT];
+}
+
+bool WaitcntBrackets::hasPendingGDS() const {
+ return LastGDS > ScoreLBs[DS_CNT] && LastGDS <= ScoreUBs[DS_CNT];
+}
+
+unsigned WaitcntBrackets::getPendingGDSWait() const {
+ return std::min(getScoreUB(DS_CNT) - LastGDS, getLimit(DS_CNT) - 1);
+}
+
+bool WaitcntBrackets::hasDifferentVGPRPendingEvents(MCPhysReg Reg,
+ HWEvents E) const {
+ for (MCRegUnit RU : regunits(Reg)) {
+ auto It = VMem.find(toVMEMID(RU));
+ if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
+ return true;
+ }
+ return false;
+}
+
+void WaitcntBrackets::clearVGPRPendingEvents(MCPhysReg Reg) {
+ for (MCRegUnit RU : regunits(Reg)) {
+ if (auto It = VMem.find(toVMEMID(RU)); It != VMem.end()) {
+ It->second.VGPRPendingEvents = HWEvents::NONE;
+ if (It->second.empty())
+ VMem.erase(It);
+ }
+ }
+}
+
+void WaitcntBrackets::setStateOnFunctionEntryOrReturn() {
+ setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getLimit(STORE_CNT));
+ PendingEvents |= getWaitEvents(STORE_CNT);
+}
+
+bool WaitcntBrackets::hasPendingEvent(InstCounterType T) const {
+ bool HasPending = (PendingEvents & getWaitEvents(T)).any();
+ assert(HasPending == !empty(T) &&
+ "Expected pending events iff scoreboard is not empty");
+ return HasPending;
+}
+
+bool WaitcntBrackets::hasMixedPendingEvents(InstCounterType T) const {
+ HWEvents Events = PendingEvents & getWaitEvents(T);
+ // Return true if more than one bit is set in Events.
+ return Events.size() > 1;
+}
+
+// Return true if the subtarget is one that enables Point Sample Acceleration
+// and the MachineInstr passed in is one to which it might be applied (the
+// hardware makes this decision based on several factors, but we can't determine
+// this at compile time, so we have to assume it might be applied if the
+// instruction supports it).
+bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
+ if (!Ctx->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
+ return false;
+
+ const MIMGInfo *Info = getMIMGInfo(MI.getOpcode());
+ const MIMGBaseOpcodeInfo *BaseInfo = getMIMGBaseOpcodeInfo(Info->BaseOpcode);
+ return BaseInfo->PointSampleAccel;
+}
+
+// Return true if the subtarget enables Point Sample Acceleration, the supplied
+// MachineInstr is one to which it might be applied and the supplied interval is
+// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
+// (this is the type that a point sample accelerated instruction effectively
+// becomes)
+bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
+ MCPhysReg Reg) const {
+ if (!hasPointSampleAccel(MI))
+ return false;
+
+ return hasDifferentVGPRPendingEvents(Reg, HWEvents::VMEM_READ_ACCESS);
+}
+
+void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
+ assert(E.size() == 1 && "Expected singular event!");
+ InstCounterType T = getCounterFromEvent(E);
+ assert(T < Ctx->MaxCounter);
+
+ const GCNSubtarget &ST = Ctx->ST;
+ const SIInstrInfo &TII = Ctx->TII;
+ const SIRegisterInfo &TRI = Ctx->TRI;
+ const MachineRegisterInfo &MRI = Ctx->MRI;
+
+ unsigned UB = getScoreUB(T);
+ unsigned Increment = 1;
+ if (T == VA_VDST && getHasMatrixScale(Inst.getOpcode()) &&
+ ST.hasVOP3PX2IncrementsVaVdstTwice()) {
+ // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
+ // two VOP3P instructions and increments VA_VDST twice.
+ Increment = 2;
+ }
+ unsigned CurrScore = UB + Increment;
+ if (CurrScore == 0)
+ report_fatal_error("InsertWaitcnt score wraparound");
+ // PendingEvents and ScoreUB need to be update regardless if this event
+ // changes the score of a register or not.
+ // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
+ PendingEvents |= E;
+ setScoreUB(T, CurrScore);
+
+ if (T == EXP_CNT) {
+ // Put score on the source vgprs. If this is a store, just use those
+ // specific register(s).
+ if (TII.isDS(Inst) && Inst.mayLoadOrStore()) {
+ // All GDS operations must protect their address register (same as
+ // export.)
+ if (const auto *AddrOp = TII.getNamedOperand(Inst, OpName::addr))
+ setScoreByOperand(*AddrOp, EXP_CNT, CurrScore);
+
+ if (Inst.mayStore()) {
+ if (const auto *Data0 = TII.getNamedOperand(Inst, OpName::data0))
+ setScoreByOperand(*Data0, EXP_CNT, CurrScore);
+ if (const auto *Data1 = TII.getNamedOperand(Inst, OpName::data1))
+ setScoreByOperand(*Data1, EXP_CNT, CurrScore);
+ } else if (SIInstrInfo::isAtomicRet(Inst) && !SIInstrInfo::isGWS(Inst) &&
+ Inst.getOpcode() != DS_APPEND &&
+ Inst.getOpcode() != DS_CONSUME &&
+ Inst.getOpcode() != DS_ORDERED_COUNT) {
+ for (const MachineOperand &Op : Inst.all_uses()) {
+ if (TRI.isVectorRegister(MRI, Op.getReg()))
+ setScoreByOperand(Op, EXP_CNT, CurrScore);
+ }
+ }
+ } else if (TII.isFLAT(Inst)) {
+ if (Inst.mayStore()) {
+ setScoreByOperand(*TII.getNamedOperand(Inst, OpName::data), EXP_CNT,
+ CurrScore);
+ } else if (SIInstrInfo::isAtomicRet(Inst)) {
+ setScoreByOperand(*TII.getNamedOperand(Inst, OpName::data), EXP_CNT,
+ CurrScore);
+ }
+ } else if (TII.isMIMG(Inst)) {
+ if (Inst.mayStore()) {
+ setScoreByOperand(Inst.getOperand(0), EXP_CNT, CurrScore);
+ } else if (SIInstrInfo::isAtomicRet(Inst)) {
+ setScoreByOperand(*TII.getNamedOperand(Inst, OpName::data), EXP_CNT,
+ CurrScore);
+ }
+ } else if (TII.isMTBUF(Inst)) {
+ if (Inst.mayStore())
+ setScoreByOperand(Inst.getOperand(0), EXP_CNT, CurrScore);
+ } else if (TII.isMUBUF(Inst)) {
+ if (Inst.mayStore()) {
+ setScoreByOperand(Inst.getOperand(0), EXP_CNT, CurrScore);
+ } else if (SIInstrInfo::isAtomicRet(Inst)) {
+ setScoreByOperand(*TII.getNamedOperand(Inst, OpName::data), EXP_CNT,
+ CurrScore);
+ }
+ } else if (TII.isLDSDIR(Inst)) {
+ // LDSDIR instructions attach the score to the destination.
+ setScoreByOperand(*TII.getNamedOperand(Inst, OpName::vdst), EXP_CNT,
+ CurrScore);
+ } else {
+ if (TII.isEXP(Inst)) {
+ // For export the destination registers are really temps that
+ // can be used as the actual source after export patching, so
+ // we need to treat them like sources and set the EXP_CNT
+ // score.
+ for (MachineOperand &DefMO : Inst.all_defs()) {
+ if (TRI.isVGPR(MRI, DefMO.getReg())) {
+ setScoreByOperand(DefMO, EXP_CNT, CurrScore);
+ }
+ }
+ }
+ for (const MachineOperand &Op : Inst.all_uses()) {
+ if (TRI.isVectorRegister(MRI, Op.getReg()))
+ setScoreByOperand(Op, EXP_CNT, CurrScore);
+ }
+ }
+ } else if (T == X_CNT) {
+ HWEvents OtherEvent =
+ E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
+ if (PendingEvents.contains(OtherEvent)) {
+ // Hardware inserts an implicit xcnt between interleaved
+ // SMEM and VMEM operations. So there will never be
+ // outstanding address translations for both SMEM and
+ // VMEM at the same time.
+ setScoreLB(T, getScoreUB(T) - 1);
+ PendingEvents -= OtherEvent;
+ }
+ for (const MachineOperand &Op : Inst.all_uses())
+ setScoreByOperand(Op, T, CurrScore);
+ } else if (T == VA_VDST || T == VM_VSRC) {
+ // Match the score to the VGPR destination or source registers as
+ // appropriate
+ for (const MachineOperand &Op : Inst.operands()) {
+ if (!Op.isReg() || (T == VA_VDST && Op.isUse()) ||
+ (T == VM_VSRC && Op.isDef()))
+ continue;
+ if (TRI.isVectorRegister(MRI, Op.getReg()))
+ setScoreByOperand(Op, T, CurrScore);
+ }
+ } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
+ // Match the score to the destination registers.
+ //
+ // Check only explicit operands. Stores, especially spill stores, include
+ // implicit uses and defs of their super registers which would create an
+ // artificial dependency, while these are there only for register liveness
+ // accounting purposes.
+ //
+ // Special cases where implicit register defs exists, such as M0 or VCC,
+ // but none with memory instructions.
+ for (const MachineOperand &Op : Inst.defs()) {
+ if (T == LOAD_CNT || T == SAMPLE_CNT || T == BVH_CNT) {
+ if (!TRI.isVectorRegister(MRI, Op.getReg()))
+ continue;
+ if (SIInstrInfo::updateVMCntOnly(Inst)) {
+ // updateVMCntOnly should only leave us with VGPRs
+ // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
+ // defs.
+ assert(TRI.isVectorRegister(MRI, Op.getReg()));
+ HWEvents VGPRContext = getSimplifiedVMEMEventsFor(Inst, TII);
+ // If instruction can have Point Sample Accel applied, we have to flag
+ // this with another potential dependency
+ if (hasPointSampleAccel(Inst))
+ VGPRContext |= HWEvents::VMEM_READ_ACCESS;
+ for (MCRegUnit RU : regunits(Op.getReg().asMCReg()))
+ VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
+ }
+ }
+ setScoreByOperand(Op, T, CurrScore);
+ }
+ if (Inst.mayStore() &&
+ (TII.isDS(Inst) || isNonAsyncLdsDmaWrite(Inst, TII))) {
+ // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
+ // written can be accessed. A load from LDS to VMEM does not need a wait.
+ //
+ // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
+ // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
+ // store. The "Slot" is the index into LDSDMAStores + 1.
+ unsigned Slot = 0;
+ for (const auto *MemOp : Inst.memoperands()) {
+ if (!MemOp->isStore() ||
+ MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
+ continue;
+ // Comparing just AA info does not guarantee memoperands are equal
+ // in general, but this is so for LDS DMA in practice.
+ auto AAI = MemOp->getAAInfo();
+ // Alias scope information gives a way to definitely identify an
+ // original memory object and practically produced in the module LDS
+ // lowering pass. If there is no scope available we will not be able
+ // to disambiguate LDS aliasing as after the module lowering all LDS
+ // is squashed into a single big object.
+ if (!AAI || !AAI.Scope)
+ break;
+ for (unsigned I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
+ for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
+ if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
+ Slot = I + 1;
+ break;
+ }
+ }
+ }
+ if (Slot)
+ break;
+ // The slot may not be valid because it can be >= NUM_LDSDMA which
+ // means the scoreboard cannot track it. We still want to preserve the
+ // MI in order to check alias information, though.
+ LDSDMAStores.push_back(&Inst);
+ Slot = LDSDMAStores.size();
+ break;
+ }
+ setVMemScore(LDSDMA_BEGIN, T, CurrScore);
+ if (Slot && Slot < NUM_LDSDMA)
+ setVMemScore(LDSDMA_BEGIN + Slot, T, CurrScore);
+ }
+
+ if (shouldUpdateAsyncMark(Inst, T, TII)) {
+ AsyncScore[T] = CurrScore;
+ }
+
+ if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
+ setRegScore(SCC, T, CurrScore);
+ PendingSCCWrite = &Inst;
+ }
+ }
+}
+
+void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
+ // In the absence of loops, AsyncMarks can grow linearly with the program
+ // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
+ // limit every time we push a new mark, but that seems like unnecessary work
+ // in practical cases. We do separately truncate the array when processing a
+ // loop, which should be sufficient.
+ AsyncMarks.push_back(AsyncScore);
+ AsyncScore = {};
+ LLVM_DEBUG({
+ dbgs() << "recordAsyncMark:\n" << Inst;
+ for (const auto &Mark : AsyncMarks) {
+ llvm::interleaveComma(Mark, dbgs());
+ dbgs() << '\n';
+ }
+ });
+}
+
+void WaitcntBrackets::print(raw_ostream &OS) const {
+ const GCNSubtarget &ST = Ctx->ST;
+ const SIRegisterInfo &TRI = Ctx->TRI;
+
+ for (auto T : inst_counter_types(Ctx->MaxCounter)) {
+ unsigned SR = getScoreRange(T);
+ switch (T) {
+ case LOAD_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
+ << SR << "):";
+ break;
+ case DS_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
+ << SR << "):";
+ break;
+ case EXP_CNT:
+ OS << " EXP_CNT(" << SR << "):";
+ break;
+ case STORE_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
+ << SR << "):";
+ break;
+ case SAMPLE_CNT:
+ OS << " SAMPLE_CNT(" << SR << "):";
+ break;
+ case BVH_CNT:
+ OS << " BVH_CNT(" << SR << "):";
+ break;
+ case KM_CNT:
+ OS << " KM_CNT(" << SR << "):";
+ break;
+ case X_CNT:
+ OS << " X_CNT(" << SR << "):";
+ break;
+ case ASYNC_CNT:
+ OS << " ASYNC_CNT(" << SR << "):";
+ break;
+ case VA_VDST:
+ OS << " VA_VDST(" << SR << "): ";
+ break;
+ case VM_VSRC:
+ OS << " VM_VSRC(" << SR << "): ";
+ break;
+ default:
+ OS << " UNKNOWN(" << SR << "):";
+ break;
+ }
+
+ if (SR != 0) {
+ // Print vgpr scores.
+ unsigned LB = getScoreLB(T);
+
+ SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
+ sort(SortedVMEMIDs);
+
+ for (auto ID : SortedVMEMIDs) {
+ unsigned RegScore = VMem.at(ID).Scores[T];
+ if (RegScore <= LB)
+ continue;
+ unsigned RelScore = RegScore - LB - 1;
+ if (ID < REGUNITS_END) {
+ OS << ' ' << RelScore << ":vRU"
----------------
jayfoad wrote:
Bad merge
```suggestion
OS << ' ' << RelScore << ':'
```
https://github.com/llvm/llvm-project/pull/210044
More information about the llvm-commits
mailing list