[llvm] [AMDGPU] Handle WAR hazards (PR #201619)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 03:04:13 PDT 2026
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/201619
>From b6595a75627a5ba3d732d8ace5c46d841b67e606 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Thu, 30 Apr 2026 16:10:25 +0100
Subject: [PATCH 1/2] [AMDGPU] Inline
SIInsertWaitcnts::getExpertSchedulingEventType. NFC.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 65 ++++++++-------------
1 file changed, 23 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7241c0db726ce..79ac2564b44d6 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -657,9 +657,6 @@ class SIInsertWaitcnts {
return VmemReadMapping[getVmemType(Inst)];
}
- std::optional<WaitEventType>
- getExpertSchedulingEventType(const MachineInstr &Inst) const;
-
bool isAsync(const MachineInstr &MI) const {
if (!SIInstrInfo::isLDSDMA(MI))
return false;
@@ -2863,43 +2860,6 @@ bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
return Modified;
}
-std::optional<WaitEventType>
-SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
- if (TII.isVALU(Inst)) {
- // Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
- // out-of-order with respect to each other, so each of these classes
- // has its own event.
-
- if (TII.isXDL(Inst))
- return VGPR_XDL_WRITE;
-
- if (TII.isTRANS(Inst))
- return VGPR_TRANS_WRITE;
-
- if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
- return VGPR_DPMACC_WRITE;
-
- return VGPR_CSMACC_WRITE;
- }
-
- // FLAT and LDS instructions may read their VGPR sources out-of-order
- // with respect to each other and all other VMEM instructions, so
- // each of these also has a separate event.
-
- if (TII.isFLAT(Inst))
- return VGPR_FLAT_READ;
-
- if (TII.isDS(Inst))
- return VGPR_LDS_READ;
-
- if (TII.isVMEM(Inst) || TII.isVIMAGE(Inst) || TII.isVSAMPLE(Inst))
- return VGPR_VMEM_READ;
-
- // Otherwise, no hazard.
-
- return {};
-}
-
bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
(TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(MI.getOpcode()));
@@ -2966,8 +2926,29 @@ bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
WaitEventSet Events;
if (IsExpertMode) {
- if (const auto ET = getExpertSchedulingEventType(Inst))
- Events.insert(*ET);
+ if (TII.isVALU(Inst)) {
+ // Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
+ // out-of-order with respect to each other, so each of these classes
+ // has its own event.
+ if (TII.isXDL(Inst))
+ Events.insert(VGPR_XDL_WRITE);
+ else if (TII.isTRANS(Inst))
+ Events.insert(VGPR_TRANS_WRITE);
+ else if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
+ Events.insert(VGPR_DPMACC_WRITE);
+ else
+ Events.insert(VGPR_CSMACC_WRITE);
+ } else {
+ // FLAT and LDS instructions may read their VGPR sources out-of-order
+ // with respect to each other and all other VMEM instructions, so
+ // each of these also has a separate event.
+ if (TII.isFLAT(Inst))
+ Events.insert(VGPR_FLAT_READ);
+ else if (TII.isDS(Inst))
+ Events.insert(VGPR_LDS_READ);
+ else if (TII.isVMEM(Inst))
+ Events.insert(VGPR_VMEM_READ);
+ }
}
if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
>From 2f06ef37c7a39ae5f5dffa1d650dc6e5d9e05eb3 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Thu, 30 Apr 2026 16:04:27 +0100
Subject: [PATCH 2/2] [AMDGPU] Handle WAR hazards
---
llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp | 6 +-
llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h | 15 ++-
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 116 +++++++++++++-----
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 +-
.../AMDGPU/expert_scheduling_gfx12.mir | 58 ++++++++-
6 files changed, 156 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
index df8d22fb5e3dd..e1f1ac2605b41 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
@@ -37,8 +37,10 @@ StringLiteral getInstCounterName(InstCounterType T) {
return "ASYNC_CNT";
case TENSOR_CNT:
return "TENSOR_CNT";
- case VA_VDST:
- return "VA_VDST";
+ case VA_VDST_RD:
+ return "VA_VDST_RD";
+ case VA_VDST_WR:
+ return "VA_VDST_WR";
case VM_VSRC:
return "VM_VSRC";
case NUM_INST_CNTS:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
index 093d8a45d207b..cdb63b0d4f9a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
@@ -32,8 +32,9 @@ enum InstCounterType {
ASYNC_CNT, // gfx1250.
TENSOR_CNT, // gfx1250.
NUM_EXTENDED_INST_CNTS,
- VA_VDST = NUM_EXTENDED_INST_CNTS, // gfx12+ expert mode only.
- VM_VSRC, // gfx12+ expert mode only.
+ VA_VDST_RD = NUM_EXTENDED_INST_CNTS, // gfx12+ expert mode only.
+ VA_VDST_WR, // gfx12+ expert mode only.
+ VM_VSRC, // gfx12+ expert mode only.
NUM_EXPERT_INST_CNTS,
NUM_INST_CNTS = NUM_EXPERT_INST_CNTS
};
@@ -78,8 +79,8 @@ class Waitcnt {
// gfx12+ constructor.
Waitcnt(unsigned LoadCnt, unsigned ExpCnt, unsigned DsCnt, unsigned StoreCnt,
unsigned SampleCnt, unsigned BvhCnt, unsigned KmCnt, unsigned XCnt,
- unsigned AsyncCnt, unsigned TensorCnt, unsigned VaVdst,
- unsigned VmVsrc)
+ unsigned AsyncCnt, unsigned TensorCnt, unsigned VaVdstRd,
+ unsigned VaVdstWr, unsigned VmVsrc)
: Waitcnt() {
Cnt[LOAD_CNT] = LoadCnt;
Cnt[DS_CNT] = DsCnt;
@@ -91,7 +92,8 @@ class Waitcnt {
Cnt[X_CNT] = XCnt;
Cnt[ASYNC_CNT] = AsyncCnt;
Cnt[TENSOR_CNT] = TensorCnt;
- Cnt[VA_VDST] = VaVdst;
+ Cnt[VA_VDST_RD] = VaVdstRd;
+ Cnt[VA_VDST_WR] = VaVdstWr;
Cnt[VM_VSRC] = VmVsrc;
}
@@ -112,7 +114,8 @@ class Waitcnt {
bool hasWaitStoreCnt() const { return Cnt[STORE_CNT] != ~0u; }
bool hasWaitDepctr() const {
- return Cnt[VA_VDST] != ~0u || Cnt[VM_VSRC] != ~0u;
+ return Cnt[VA_VDST_RD] != ~0u || Cnt[VA_VDST_WR] != ~0u ||
+ Cnt[VM_VSRC] != ~0u;
}
Waitcnt combined(const Waitcnt &Other) const {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 79ac2564b44d6..a50b047e8410b 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -90,8 +90,10 @@ static unsigned getWaitCountMax(const AMDGPU::HardwareLimits &Limits,
return Limits.KmcntMax;
case AMDGPU::X_CNT:
return Limits.XcntMax;
- case AMDGPU::VA_VDST:
- return Limits.VaVdstMax;
+ case AMDGPU::VA_VDST_RD:
+ return Limits.VaVdstRdMax;
+ case AMDGPU::VA_VDST_WR:
+ return Limits.VaVdstWrMax;
case AMDGPU::VM_VSRC:
return Limits.VmVsrcMax;
default:
@@ -154,6 +156,10 @@ static constexpr VMEMID toVMEMID(MCRegUnit RU) {
DECL(EXP_PARAM_ACCESS) /* write to export parameter */ \
DECL(VMW_GPR_LOCK) /* vmem write holding on its data src */ \
DECL(EXP_LDS_ACCESS) /* read by ldsdir counting as export */ \
+ DECL(VGPR_CSMACC_READ) /* read VGPR source in Core/Side-MACC VALU */ \
+ DECL(VGPR_DPMACC_READ) /* read VGPR source in DPMACC VALU */ \
+ DECL(VGPR_TRANS_READ) /* read VGPR source in TRANS VALU */ \
+ DECL(VGPR_XDL_READ) /* read VGPR source in XDL VALU */ \
DECL(VGPR_CSMACC_WRITE) /* write VGPR dest in Core/Side-MACC VALU */ \
DECL(VGPR_DPMACC_WRITE) /* write VGPR dest in DPMACC VALU */ \
DECL(VGPR_TRANS_WRITE) /* write VGPR dest in TRANS VALU */ \
@@ -215,7 +221,7 @@ enum VmemType {
// Maps values of InstCounterType to the instruction that waits on that
// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
-// returns true, and does not cover VA_VDST or VM_VSRC.
+// returns true, and does not cover VA_VDST_RD, VA_VDST_WR or VM_VSRC.
static const unsigned
instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
@@ -487,6 +493,8 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
WaitEventSet({VMEM_GROUP, SMEM_GROUP}),
WaitEventSet({ASYNC_ACCESS}),
WaitEventSet({TENSOR_ACCESS}),
+ WaitEventSet({VGPR_CSMACC_READ, VGPR_DPMACC_READ, VGPR_TRANS_READ,
+ VGPR_XDL_READ}),
WaitEventSet({VGPR_CSMACC_WRITE, VGPR_DPMACC_WRITE, VGPR_TRANS_WRITE,
VGPR_XDL_WRITE}),
WaitEventSet({VGPR_LDS_READ, VGPR_FLAT_READ, VGPR_VMEM_READ})};
@@ -617,7 +625,8 @@ class SIInsertWaitcnts {
ForceEmitWaitcnt[AMDGPU::BVH_CNT] = false;
}
- ForceEmitWaitcnt[AMDGPU::VA_VDST] = false;
+ ForceEmitWaitcnt[AMDGPU::VA_VDST_RD] = false;
+ ForceEmitWaitcnt[AMDGPU::VA_VDST_WR] = false;
ForceEmitWaitcnt[AMDGPU::VM_VSRC] = false;
#endif // NDEBUG
}
@@ -1124,7 +1133,8 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
unsigned UB = getScoreUB(T);
unsigned Increment = 1;
- if (T == AMDGPU::VA_VDST && AMDGPU::getHasMatrixScale(Inst.getOpcode())) {
+ if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
+ AMDGPU::getHasMatrixScale(Inst.getOpcode())) {
// V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
// two VOP3P instructions and increments VA_VDST twice.
Increment = 2;
@@ -1225,13 +1235,24 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
}
for (const MachineOperand &Op : Inst.all_uses())
setScoreByOperand(Op, T, CurrScore);
- } else if (T == AMDGPU::VA_VDST || T == AMDGPU::VM_VSRC) {
- // Match the score to the VGPR destination or source registers as
- // appropriate
+ } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
+ T == AMDGPU::VM_VSRC) {
+ // Match the score to VGPR registers based on counter type:
+ // VA_VDST_RD: Track VGPR defs (writes) - wait before reads
+ // VA_VDST_WR: Track VGPR uses (reads) - wait before writes
+ // VM_VSRC: Track VGPR uses (reads) - wait before reads
for (const MachineOperand &Op : Inst.operands()) {
- if (!Op.isReg() || (T == AMDGPU::VA_VDST && Op.isUse()) ||
- (T == AMDGPU::VM_VSRC && Op.isDef()))
+ if (!Op.isReg())
continue;
+
+ // Skip based on counter type and operand type
+ if (T == AMDGPU::VA_VDST_RD && Op.isDef())
+ continue; // RD tracks reads only
+ if (T == AMDGPU::VA_VDST_WR && Op.isUse())
+ continue; // WR tracks writes only
+ if (T == AMDGPU::VM_VSRC && Op.isDef())
+ continue;
+
if (TRI.isVectorRegister(Context->MRI, Op.getReg()))
setScoreByOperand(Op, T, CurrScore);
}
@@ -1376,8 +1397,11 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
case AMDGPU::ASYNC_CNT:
OS << " ASYNC_CNT(" << SR << "):";
break;
- case AMDGPU::VA_VDST:
- OS << " VA_VDST(" << SR << "): ";
+ case AMDGPU::VA_VDST_RD:
+ OS << " VA_VDST_RD(" << SR << "): ";
+ break;
+ case AMDGPU::VA_VDST_WR:
+ OS << " VA_VDST_WR(" << SR << "): ";
break;
case AMDGPU::VM_VSRC:
OS << " VM_VSRC(" << SR << "): ";
@@ -1505,7 +1529,8 @@ void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
simplifyWaitcnt(UpdateWait, AMDGPU::BVH_CNT);
simplifyWaitcnt(UpdateWait, AMDGPU::KM_CNT);
simplifyXcnt(CheckWait, UpdateWait);
- simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST);
+ simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_RD);
+ simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_WR);
simplifyVmVsrc(CheckWait, UpdateWait);
simplifyWaitcnt(UpdateWait, AMDGPU::ASYNC_CNT);
}
@@ -2097,7 +2122,8 @@ WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
~0u /* XCNT */, ~0u /* ASYNC_CNT */,
- ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal);
+ ~0u /* TENSOR_CNT */, ExpertVal /* VA_VDST_RD */,
+ ExpertVal /* VA_VDST_WR */, ExpertVal /* VM_VSRC */);
}
/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
@@ -2179,7 +2205,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
unsigned OldEnc =
TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
AMDGPU::Waitcnt OldWait;
- OldWait.set(AMDGPU::VA_VDST, AMDGPU::DepCtr::decodeFieldVaVdst(OldEnc));
+ // Set both counters to the decoded value from the single hardware field
+ unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(OldEnc);
+ OldWait.set(AMDGPU::VA_VDST_RD, VaVdst);
+ OldWait.set(AMDGPU::VA_VDST_WR, VaVdst);
OldWait.set(AMDGPU::VM_VSRC, AMDGPU::DepCtr::decodeFieldVmVsrc(OldEnc));
if (TrySimplify)
ScoreBrackets.simplifyWaitcnt(OldWait);
@@ -2188,8 +2217,8 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
WaitcntDepctrInstr = &II;
} else {
// S_WAITCNT_DEPCTR requires special care. Don't remove a
- // duplicate if it is waiting on things other than VA_VDST or
- // VM_VSRC. If that is the case, just make sure the VA_VDST and
+ // duplicate if it is waiting on things other than VA_VDST_RD/WR or
+ // VM_VSRC. If that is the case, just make sure the VA_VDST_RD/WR and
// VM_VSRC subfields of the operand are set to the "no wait"
// values.
@@ -2358,17 +2387,24 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
}
if (WaitcntDepctrInstr) {
- // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
- // subfields with the new required values.
+ // Get the encoded Depctr immediate and override the VA_VDST_RD/WR and
+ // VM_VSRC subfields with the new required values.
unsigned Enc =
TII.getNamedOperand(*WaitcntDepctrInstr, AMDGPU::OpName::simm16)
->getImm();
Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Enc, Wait.get(AMDGPU::VM_VSRC));
- Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, Wait.get(AMDGPU::VA_VDST));
-
- ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST, Wait.get(AMDGPU::VA_VDST));
+ // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
+ unsigned VaVdst =
+ std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
+ Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
+
+ ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_RD,
+ Wait.get(AMDGPU::VA_VDST_RD));
+ ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_WR,
+ Wait.get(AMDGPU::VA_VDST_WR));
ScoreBrackets.applyWaitcnt(AMDGPU::VM_VSRC, Wait.get(AMDGPU::VM_VSRC));
- Wait.set(AMDGPU::VA_VDST, ~0u);
+ Wait.set(AMDGPU::VA_VDST_RD, ~0u);
+ Wait.set(AMDGPU::VA_VDST_WR, ~0u);
Wait.set(AMDGPU::VM_VSRC, ~0u);
// If that new encoded Depctr immediate would actually still wait
@@ -2491,7 +2527,10 @@ bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
assert(IsExpertMode);
unsigned Enc =
AMDGPU::DepCtr::encodeFieldVmVsrc(Wait.get(AMDGPU::VM_VSRC), ST);
- Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, Wait.get(AMDGPU::VA_VDST));
+ // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
+ unsigned VaVdst =
+ std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
+ Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
[[maybe_unused]] auto SWaitInst =
BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_DEPCTR)).addImm(Enc);
@@ -2708,10 +2747,14 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (Op.isImplicit() && MI.mayLoadOrStore())
continue;
- ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST, Reg, Wait, MI);
- if (Op.isDef())
+ ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_WR, Reg, Wait,
+ MI);
+ if (Op.isDef()) {
+ ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_RD, Reg, Wait,
+ MI);
ScoreBrackets.determineWaitForPhysReg(AMDGPU::VM_VSRC, Reg, Wait,
MI);
+ }
// RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
// previous write and this write are the same type of VMEM
// instruction, in which case they are (in some architectures)
@@ -2779,11 +2822,13 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
ScoreBrackets.simplifyWaitcnt(Wait);
// It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
- // waits on VA_VDST if the instruction it would precede is not a VALU
+ // waits on VA_VDST_RD/WR if the instruction it would precede is not a VALU
// instruction, since hardware handles VALU->VGPR->VALU hazards in
// expert scheduling mode.
- if (TII.isVALU(MI))
- Wait.set(AMDGPU::VA_VDST, ~0u);
+ if (TII.isVALU(MI)) {
+ Wait.set(AMDGPU::VA_VDST_RD, ~0u);
+ Wait.set(AMDGPU::VA_VDST_WR, ~0u);
+ }
// Since the translation for VMEM addresses occur in-order, we can apply the
// XCnt if the current instruction is of VMEM type and has a memory
@@ -2930,14 +2975,19 @@ WaitEventSet SIInsertWaitcnts::getEventsFor(const MachineInstr &Inst) const {
// Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
// out-of-order with respect to each other, so each of these classes
// has its own event.
- if (TII.isXDL(Inst))
+ if (TII.isXDL(Inst)) {
+ Events.insert(VGPR_XDL_READ);
Events.insert(VGPR_XDL_WRITE);
- else if (TII.isTRANS(Inst))
+ } else if (TII.isTRANS(Inst)) {
+ Events.insert(VGPR_TRANS_READ);
Events.insert(VGPR_TRANS_WRITE);
- else if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
+ } else if (AMDGPU::isDPMACCInstruction(Inst.getOpcode())) {
+ Events.insert(VGPR_DPMACC_READ);
Events.insert(VGPR_DPMACC_WRITE);
- else
+ } else {
+ Events.insert(VGPR_CSMACC_READ);
Events.insert(VGPR_CSMACC_WRITE);
+ }
} else {
// FLAT and LDS instructions may read their VGPR sources out-of-order
// with respect to each other and all other VMEM instructions, so
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 33df51e8a7e07..4f315190ea443 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1851,7 +1851,8 @@ HardwareLimits::HardwareLimits(const IsaVersion &IV) {
KmcntMax = getKmcntBitMask(IV);
XcntMax = getXcntBitMask(IV);
AsyncMax = getAsynccntBitMask(IV);
- VaVdstMax = DepCtr::getVaVdstBitMask();
+ VaVdstRdMax = DepCtr::getVaVdstBitMask();
+ VaVdstWrMax = DepCtr::getVaVdstBitMask();
VmVsrcMax = DepCtr::getVmVsrcBitMask();
}
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 1f7084c8d25ae..4cf52bfb52e54 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1112,7 +1112,8 @@ struct HardwareLimits {
unsigned KmcntMax; // gfx12+ only.
unsigned XcntMax; // gfx1250.
unsigned AsyncMax; // gfx1250.
- unsigned VaVdstMax; // gfx12+ expert mode only.
+ unsigned VaVdstRdMax; // gfx12+ expert mode only.
+ unsigned VaVdstWrMax; // gfx12+ expert mode only.
unsigned VmVsrcMax; // gfx12+ expert mode only.
HardwareLimits() = default;
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
index 7eabf8c530260..705d1d19c3809 100644
--- a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
@@ -49,6 +49,7 @@ body: |
; GCN-NEXT: S_WAIT_BVHCNT 0
; GCN-NEXT: S_WAIT_KMCNT 0
; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_LOAD_V4_V1_gfx12 $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr3, implicit $mode, implicit $exec
@@ -58,7 +59,7 @@ body: |
; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr0, implicit $mode, implicit $exec
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: IMAGE_STORE_V4_V1_gfx12 $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
- ; GCN-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0_VmVsrc_0
; GCN-NEXT: S_SETREG_IMM32_B32 0, 2074, implicit-def $mode, implicit $mode
; GCN-NEXT: SI_RETURN_TO_EPILOG $vgpr0, $vgpr1, $vgpr2, $vgpr3
$vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
@@ -126,6 +127,7 @@ body: |
; GCN-NEXT: S_WAIT_KMCNT 0
; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 $vgpr2, implicit $exec, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 $vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
; GCN-NEXT: S_WAIT_DSCNT 0
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
@@ -207,7 +209,7 @@ body: |
; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr0, implicit $mode, implicit $exec
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: IMAGE_STORE_V4_V1_gfx12 $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
- ; GCN-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0_VmVsrc_0
; GCN-NEXT: S_SETREG_IMM32_B32 0, 2074, implicit-def $mode, implicit $mode
; GCN-NEXT: SI_RETURN_TO_EPILOG $vgpr0, $vgpr1, $vgpr2, $vgpr3
$vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
@@ -684,3 +686,55 @@ body: |
$vgpr2 = V_MOV_B32_e32 $vgpr2, implicit $exec
$vgpr0 = V_MOV_B32_e32 0, implicit $exec
...
+
+---
+# Test WAR hazard: VALU source registers must be tracked for VA_VDST.
+# When a memory instruction (ds_load/vmem) overwrites VALU source registers,
+# we need s_waitcnt_depctr to ensure the VALU has finished reading them.
+# This applies to all VALU instructions, not just WMMA/XDL.
+name: test_valu_src_war_hazard_ds_load
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr20
+
+ ; GCN-LABEL: name: test_valu_src_war_hazard_ds_load
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr20
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; GCN-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GCN-NEXT: S_WAIT_EXPCNT 0
+ ; GCN-NEXT: S_WAIT_SAMPLECNT 0
+ ; GCN-NEXT: S_WAIT_BVHCNT 0
+ ; GCN-NEXT: S_WAIT_KMCNT 0
+ ; GCN-NEXT: early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, $vgpr10_vgpr11_vgpr12_vgpr13, 8, $vgpr14_vgpr15_vgpr16_vgpr17, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; GCN-NEXT: $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128_gfx9 $vgpr20, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, $vgpr10_vgpr11_vgpr12_vgpr13, 8, $vgpr14_vgpr15_vgpr16_vgpr17, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, implicit $exec
+ $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128_gfx9 $vgpr20, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+...
+
+---
+# Test WAR hazard with regular VALU (not WMMA): when ds_load overwrites
+# a register that was just read by V_ADD, we need va_vdst wait.
+name: test_regular_valu_src_war_hazard
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr10
+
+ ; GCN-LABEL: name: test_regular_valu_src_war_hazard
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr10
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; GCN-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GCN-NEXT: S_WAIT_EXPCNT 0
+ ; GCN-NEXT: S_WAIT_SAMPLECNT 0
+ ; GCN-NEXT: S_WAIT_BVHCNT 0
+ ; GCN-NEXT: S_WAIT_KMCNT 0
+ ; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec
+ ; GCN-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; GCN-NEXT: $vgpr1_vgpr2_vgpr3_vgpr4 = DS_READ_B128_gfx9 $vgpr10, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ $vgpr0 = nofpexcept V_ADD_F32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec
+ $vgpr5 = nofpexcept V_ADD_F32_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec
+ $vgpr1_vgpr2_vgpr3_vgpr4 = DS_READ_B128_gfx9 $vgpr10, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+...
More information about the llvm-commits
mailing list