[llvm] [AMDGPU] Handle WAR hazards (PR #201619)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 03:05:13 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Jay Foad (jayfoad)
<details>
<summary>Changes</summary>
---
Patch is 21.28 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/201619.diff
7 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp (+4-4)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def (+13-9)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h (+1-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp (+6-3)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h (+9-6)
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+59-19)
- (modified) llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir (+56-2)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
index 1b362aa8c8bd5..ffadc9910c4ef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.cpp
@@ -28,15 +28,15 @@ static HWEvents getExpertSchedulingEventType(const MachineInstr &Inst,
// has its own event.
if (TII.isXDL(Inst))
- return HWEvents::VGPR_XDL_WRITE;
+ return HWEvents::VGPR_XDL_READ | HWEvents::VGPR_XDL_WRITE;
if (TII.isTRANS(Inst))
- return HWEvents::VGPR_TRANS_WRITE;
+ return HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_TRANS_WRITE;
if (AMDGPU::isDPMACCInstruction(Inst.getOpcode()))
- return HWEvents::VGPR_DPMACC_WRITE;
+ return HWEvents::VGPR_DPMACC_READ | HWEvents::VGPR_DPMACC_WRITE;
- return HWEvents::VGPR_CSMACC_WRITE;
+ return HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_CSMACC_WRITE;
}
// FLAT and LDS instructions may read their VGPR sources out-of-order
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
index bd12af16d66cc..dc97c0b8ae1f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.def
@@ -38,15 +38,19 @@ AMDGPU_HW_EVENT(EXP_POS_ACCESS, 15) /* write to export position */
AMDGPU_HW_EVENT(EXP_PARAM_ACCESS, 16) /* write to export parameter */
AMDGPU_HW_EVENT(VMW_GPR_LOCK, 17) /* vmem write holding on its data src */
AMDGPU_HW_EVENT(EXP_LDS_ACCESS, 18) /* read by ldsdir counting as export */
-AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE, 19) /* write VGPR dest in Core/Side-MACC VALU */
-AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE, 20) /* write VGPR dest in DPMACC VALU */
-AMDGPU_HW_EVENT(VGPR_TRANS_WRITE, 21) /* write VGPR dest in TRANS VALU */
-AMDGPU_HW_EVENT(VGPR_XDL_WRITE, 22) /* write VGPR dest in XDL VALU */
-AMDGPU_HW_EVENT(VGPR_LDS_READ, 23) /* read VGPR source in LDS */
-AMDGPU_HW_EVENT(VGPR_FLAT_READ, 24) /* read VGPR source in FLAT */
-AMDGPU_HW_EVENT(VGPR_VMEM_READ, 25) /* read VGPR source in other VMEM */
-AMDGPU_HW_EVENT(ASYNC_ACCESS, 26) /* access that uses ASYNC_CNT */
-AMDGPU_HW_EVENT(TENSOR_ACCESS, 27) /* access that uses TENSOR_CNT */
+AMDGPU_HW_EVENT(VGPR_CSMACC_READ, 19) /* read VGPR source in Core/Side-MACC VALU */
+AMDGPU_HW_EVENT(VGPR_DPMACC_READ, 20) /* read VGPR source in DPMACC VALU */
+AMDGPU_HW_EVENT(VGPR_TRANS_READ, 21) /* read VGPR source in TRANS VALU */
+AMDGPU_HW_EVENT(VGPR_XDL_READ, 22) /* read VGPR source in XDL VALU */
+AMDGPU_HW_EVENT(VGPR_CSMACC_WRITE, 23) /* write VGPR dest in Core/Side-MACC VALU */
+AMDGPU_HW_EVENT(VGPR_DPMACC_WRITE, 24) /* write VGPR dest in DPMACC VALU */
+AMDGPU_HW_EVENT(VGPR_TRANS_WRITE, 25) /* write VGPR dest in TRANS VALU */
+AMDGPU_HW_EVENT(VGPR_XDL_WRITE, 26) /* write VGPR dest in XDL VALU */
+AMDGPU_HW_EVENT(VGPR_LDS_READ, 27) /* read VGPR source in LDS */
+AMDGPU_HW_EVENT(VGPR_FLAT_READ, 28) /* read VGPR source in FLAT */
+AMDGPU_HW_EVENT(VGPR_VMEM_READ, 29) /* read VGPR source in other VMEM */
+AMDGPU_HW_EVENT(ASYNC_ACCESS, 30) /* access that uses ASYNC_CNT */
+AMDGPU_HW_EVENT(TENSOR_ACCESS, 31) /* access that uses TENSOR_CNT */
AMDGPU_LAST_HW_EVENT(TENSOR_ACCESS)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
index aa0a89e498ae1..eb25206b5ee04 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHWEvents.h
@@ -53,7 +53,7 @@ class HWEvents {
enum : value_type {
NONE = 0,
-#define AMDGPU_HW_EVENT(X, V) X = (1 << V),
+#define AMDGPU_HW_EVENT(X, V) X = (1u << V),
#define AMDGPU_LAST_HW_EVENT(X) HWEVENT_LAST_EVENT = X,
#include "AMDGPUHWEvents.def"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
index 75e757d0c8b2d..43896675621f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.cpp
@@ -38,8 +38,10 @@ StringLiteral getInstCounterName(InstCounterType T) {
return "ASYNC_CNT";
case TENSOR_CNT:
return "TENSOR_CNT";
- case VA_VDST:
- return "VA_VDST";
+ case VA_VDST_RD:
+ return "VA_VDST_RD";
+ case VA_VDST_WR:
+ return "VA_VDST_WR";
case VM_VSRC:
return "VM_VSRC";
case NUM_INST_CNTS:
@@ -86,7 +88,8 @@ unsigned HardwareLimits::get(InstCounterType T) const {
return KmcntMax;
case AMDGPU::X_CNT:
return XcntMax;
- case AMDGPU::VA_VDST:
+ case AMDGPU::VA_VDST_RD:
+ case AMDGPU::VA_VDST_WR:
return VaVdstMax;
case AMDGPU::VM_VSRC:
return VmVsrcMax;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
index 0930a95288087..827494e264c50 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitcntUtils.h
@@ -32,8 +32,9 @@ enum InstCounterType {
ASYNC_CNT, // gfx1250.
TENSOR_CNT, // gfx1250.
NUM_EXTENDED_INST_CNTS,
- VA_VDST = NUM_EXTENDED_INST_CNTS, // gfx12+ expert mode only.
- VM_VSRC, // gfx12+ expert mode only.
+ VA_VDST_RD = NUM_EXTENDED_INST_CNTS, // gfx12+ expert mode only.
+ VA_VDST_WR, // gfx12+ expert mode only.
+ VM_VSRC, // gfx12+ expert mode only.
NUM_EXPERT_INST_CNTS,
NUM_INST_CNTS = NUM_EXPERT_INST_CNTS
};
@@ -100,8 +101,8 @@ class Waitcnt {
// gfx12+ constructor.
Waitcnt(unsigned LoadCnt, unsigned ExpCnt, unsigned DsCnt, unsigned StoreCnt,
unsigned SampleCnt, unsigned BvhCnt, unsigned KmCnt, unsigned XCnt,
- unsigned AsyncCnt, unsigned TensorCnt, unsigned VaVdst,
- unsigned VmVsrc)
+ unsigned AsyncCnt, unsigned TensorCnt, unsigned VaVdstRd,
+ unsigned VaVdstWr, unsigned VmVsrc)
: Waitcnt() {
Cnt[LOAD_CNT] = LoadCnt;
Cnt[DS_CNT] = DsCnt;
@@ -113,7 +114,8 @@ class Waitcnt {
Cnt[X_CNT] = XCnt;
Cnt[ASYNC_CNT] = AsyncCnt;
Cnt[TENSOR_CNT] = TensorCnt;
- Cnt[VA_VDST] = VaVdst;
+ Cnt[VA_VDST_RD] = VaVdstRd;
+ Cnt[VA_VDST_WR] = VaVdstWr;
Cnt[VM_VSRC] = VmVsrc;
}
@@ -140,7 +142,8 @@ class Waitcnt {
bool hasWaitStoreCnt() const { return Cnt[STORE_CNT] != ~0u; }
bool hasWaitDepctr() const {
- return Cnt[VA_VDST] != ~0u || Cnt[VM_VSRC] != ~0u;
+ return Cnt[VA_VDST_RD] != ~0u || Cnt[VA_VDST_WR] != ~0u ||
+ Cnt[VM_VSRC] != ~0u;
}
Waitcnt combined(const Waitcnt &Other) const {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 5996cdf42a956..fdf8b20cea34f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -254,6 +254,7 @@ class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
HWEvents::NONE,
HWEvents::NONE,
HWEvents::NONE,
+ HWEvents::NONE,
HWEvents::NONE};
public:
@@ -297,6 +298,8 @@ class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
HWEvents::ASYNC_ACCESS,
HWEvents::TENSOR_ACCESS,
+ HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
+ HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
@@ -865,7 +868,8 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
unsigned UB = getScoreUB(T);
unsigned Increment = 1;
- if (T == AMDGPU::VA_VDST && AMDGPU::getHasMatrixScale(Inst.getOpcode()) &&
+ if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
+ AMDGPU::getHasMatrixScale(Inst.getOpcode()) &&
Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
// V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
// two VOP3P instructions and increments VA_VDST twice.
@@ -968,13 +972,22 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
}
for (const MachineOperand &Op : Inst.all_uses())
setScoreByOperand(Op, T, CurrScore);
- } else if (T == AMDGPU::VA_VDST || T == AMDGPU::VM_VSRC) {
+ } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
+ T == AMDGPU::VM_VSRC) {
// Match the score to the VGPR destination or source registers as
// appropriate
for (const MachineOperand &Op : Inst.operands()) {
- if (!Op.isReg() || (T == AMDGPU::VA_VDST && Op.isUse()) ||
- (T == AMDGPU::VM_VSRC && Op.isDef()))
+ if (!Op.isReg())
+ continue;
+
+ // Skip based on counter type and operand type
+ if (T == AMDGPU::VA_VDST_RD && Op.isDef())
+ continue; // RD tracks reads only
+ if (T == AMDGPU::VA_VDST_WR && Op.isUse())
+ continue; // WR tracks writes only
+ if (T == AMDGPU::VM_VSRC && Op.isDef())
continue;
+
if (TRI.isVectorRegister(Context->MRI, Op.getReg()))
setScoreByOperand(Op, T, CurrScore);
}
@@ -1119,8 +1132,11 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
case AMDGPU::ASYNC_CNT:
OS << " ASYNC_CNT(" << SR << "):";
break;
- case AMDGPU::VA_VDST:
- OS << " VA_VDST(" << SR << "): ";
+ case AMDGPU::VA_VDST_RD:
+ OS << " VA_VDST_RD(" << SR << "): ";
+ break;
+ case AMDGPU::VA_VDST_WR:
+ OS << " VA_VDST_WR(" << SR << "): ";
break;
case AMDGPU::VM_VSRC:
OS << " VM_VSRC(" << SR << "): ";
@@ -1243,7 +1259,8 @@ void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
simplifyWaitcnt(UpdateWait, AMDGPU::BVH_CNT);
simplifyWaitcnt(UpdateWait, AMDGPU::KM_CNT);
simplifyXcnt(CheckWait, UpdateWait);
- simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST);
+ simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_RD);
+ simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_WR);
simplifyVmVsrc(CheckWait, UpdateWait);
simplifyWaitcnt(UpdateWait, AMDGPU::ASYNC_CNT);
}
@@ -1818,7 +1835,7 @@ WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
~0u /* XCNT */, ~0u /* ASYNC_CNT */,
- ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal);
+ ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
}
/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
@@ -1900,7 +1917,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
unsigned OldEnc =
TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
AMDGPU::Waitcnt OldWait;
- OldWait.set(AMDGPU::VA_VDST, AMDGPU::DepCtr::decodeFieldVaVdst(OldEnc));
+ // Set both counters to the decoded value from the single hardware field
+ unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(OldEnc);
+ OldWait.set(AMDGPU::VA_VDST_RD, VaVdst);
+ OldWait.set(AMDGPU::VA_VDST_WR, VaVdst);
OldWait.set(AMDGPU::VM_VSRC, AMDGPU::DepCtr::decodeFieldVmVsrc(OldEnc));
if (TrySimplify)
ScoreBrackets.simplifyWaitcnt(OldWait);
@@ -1917,7 +1937,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
unsigned Enc =
TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Enc, ~0u);
- Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, ~0u);
+ // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
+ unsigned VaVdst = std::min(Wait.get(AMDGPU::VA_VDST_RD),
+ Wait.get(AMDGPU::VA_VDST_WR));
+ Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(ST)) {
Modified |= updateOperandIfDifferent(II, AMDGPU::OpName::simm16, Enc);
@@ -2086,11 +2109,18 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
TII.getNamedOperand(*WaitcntDepctrInstr, AMDGPU::OpName::simm16)
->getImm();
Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Enc, Wait.get(AMDGPU::VM_VSRC));
- Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, Wait.get(AMDGPU::VA_VDST));
-
- ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST, Wait.get(AMDGPU::VA_VDST));
+ // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
+ unsigned VaVdst =
+ std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
+ Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
+
+ ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_RD,
+ Wait.get(AMDGPU::VA_VDST_RD));
+ ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_WR,
+ Wait.get(AMDGPU::VA_VDST_WR));
ScoreBrackets.applyWaitcnt(AMDGPU::VM_VSRC, Wait.get(AMDGPU::VM_VSRC));
- Wait.set(AMDGPU::VA_VDST, ~0u);
+ Wait.set(AMDGPU::VA_VDST_RD, ~0u);
+ Wait.set(AMDGPU::VA_VDST_WR, ~0u);
Wait.set(AMDGPU::VM_VSRC, ~0u);
// If that new encoded Depctr immediate would actually still wait
@@ -2205,7 +2235,10 @@ bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
assert(IsExpertMode);
unsigned Enc =
AMDGPU::DepCtr::encodeFieldVmVsrc(Wait.get(AMDGPU::VM_VSRC), ST);
- Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, Wait.get(AMDGPU::VA_VDST));
+ // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
+ unsigned VaVdst =
+ std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
+ Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
[[maybe_unused]] auto SWaitInst =
BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_DEPCTR)).addImm(Enc);
@@ -2422,10 +2455,15 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (Op.isImplicit() && MI.mayLoadOrStore())
continue;
- ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST, Reg, Wait, MI);
- if (Op.isDef())
+ ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_WR, Reg, Wait,
+ MI);
+ if (Op.isDef()) {
+ ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_RD, Reg, Wait,
+ MI);
ScoreBrackets.determineWaitForPhysReg(AMDGPU::VM_VSRC, Reg, Wait,
MI);
+ }
+
// RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
// previous write and this write are the same type of VMEM
// instruction, in which case they are (in some architectures)
@@ -2498,8 +2536,10 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// waits on VA_VDST if the instruction it would precede is not a VALU
// instruction, since hardware handles VALU->VGPR->VALU hazards in
// expert scheduling mode.
- if (TII.isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isLDSDMA(MI))
- Wait.set(AMDGPU::VA_VDST, ~0u);
+ if (TII.isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isLDSDMA(MI)) {
+ Wait.set(AMDGPU::VA_VDST_RD, ~0u);
+ Wait.set(AMDGPU::VA_VDST_WR, ~0u);
+ }
// Since the translation for VMEM addresses occur in-order, we can apply the
// XCnt if the current instruction is of VMEM type and has a memory
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
index 6f320f538440b..4f15917679467 100644
--- a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx12.mir
@@ -49,6 +49,7 @@ body: |
; GCN-NEXT: S_WAIT_BVHCNT 0
; GCN-NEXT: S_WAIT_KMCNT 0
; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_LOAD_V4_V1_gfx12 $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr3, implicit $mode, implicit $exec
@@ -58,7 +59,7 @@ body: |
; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr0, implicit $mode, implicit $exec
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: IMAGE_STORE_V4_V1_gfx12 $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
- ; GCN-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0_VmVsrc_0
; GCN-NEXT: S_SETREG_IMM32_B32 0, 2074, implicit-def $mode, implicit $mode
; GCN-NEXT: SI_RETURN_TO_EPILOG $vgpr0, $vgpr1, $vgpr2, $vgpr3
$vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
@@ -126,6 +127,7 @@ body: |
; GCN-NEXT: S_WAIT_KMCNT 0
; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 $vgpr2, implicit $exec, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 $vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
; GCN-NEXT: S_WAIT_DSCNT 0
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
@@ -207,7 +209,7 @@ body: |
; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 1065353216, $vgpr0, implicit $mode, implicit $exec
; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
; GCN-NEXT: IMAGE_STORE_V4_V1_gfx12 $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
- ; GCN-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0_VmVsrc_0
; GCN-NEXT: S_SETREG_IMM32_B32 0, 2074, implicit-def $mode, implicit $mode
; GCN-NEXT: SI_RETURN_TO_EPILOG $vgpr0, $vgpr1, $vgpr2, $vgpr3
$vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
@@ -684,3 +686,55 @@ body: |
$vgpr2 = V_MOV_B32_e32 $vgpr2, implicit $exec
$vgpr0 = V_MOV_B32_e32 0, implicit $exec
...
+
+---
+# Test WAR hazard: VALU source registers must be tracked for VA_VDST.
+# When a memory instruction (ds_load/vmem) overwrites VALU source registers,
+# we need s_waitcnt_depctr to ensure the VALU has finished reading them.
+# This applies to all VALU instructions, not just WMMA/XDL.
+name: test_valu_src_war_hazard_ds_load
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr20
+
+ ; GCN-LABEL: name: test_valu_src_war_hazard_ds_load
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr20
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; GCN-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GCN-NEXT: S_WAIT_EXPCNT 0
+ ; GCN-NEXT: S_WAIT_SAMPLECNT 0
+ ; GCN-NEXT: S_WAIT_BVHCNT 0
+ ; GCN-NEXT: S_WAIT_KMCNT 0
+ ; GCN-NEXT: early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, $vgpr10_vgpr11_vgpr12_vgpr13, 8, $vgpr14_vgpr15_vgpr16_vgpr17, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, implicit $exec
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; GCN-NEXT: $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128_gfx9 $vgpr20, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, $vgpr10_vgpr11_vgpr12_vgpr13, 8, $vgpr14_vgpr15_vgpr16_vgpr17, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, implicit $exec
+ $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128_gfx9 $vgpr20, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+...
+
+---
+# Test ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/201619
More information about the llvm-commits
mailing list