[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Nov 3 13:48:38 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Yaxun (Sam) Liu (yxsamliu)
<details>
<summary>Changes</summary>
Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.
Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.
---
Patch is 76.11 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/238539.diff
20 Files Affected:
- (added) clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c (+20)
- (added) clang/test/Driver/amdgpu-async-barrier-replay-warning.s (+15)
- (modified) llvm/docs/AMDGPUUsage.md (+28)
- (modified) llvm/docs/ReleaseNotes.md (+6)
- (modified) llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp (+81)
- (modified) llvm/lib/Target/AMDGPU/GCNFeatures.td (+7)
- (modified) llvm/lib/Target/AMDGPU/GCNSubtarget.h (+6)
- (modified) llvm/lib/Target/AMDGPU/SIFrameLowering.cpp (+28-8)
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+38)
- (modified) llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp (+8-3)
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir (+32)
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll (+494)
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll (+46)
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir (+269)
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll (+24)
- (modified) llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll (+12)
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s (+38)
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s (+27)
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning.s (+187)
``````````diff
diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
new file mode 100644
index 00000000000000..b0b57f42c26bd0
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
@@ -0,0 +1,20 @@
+// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
+
+void unprotected(void) {
+ __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
+// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+}
+
+void protected(void) {
+ __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
+ "s_mov_b32 s7, exec_lo\n"
+ "s_mov_b32 exec_lo, 0\n"
+ "global_load_async_to_lds_b8 v0, v[0:1], off\n"
+ "s_mov_b32 exec_lo, s7"
+ ::: "v0", "v1", "s7", "memory");
+}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..3a1549c3b325e3
--- /dev/null
+++ b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
@@ -0,0 +1,15 @@
+// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
+// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
+// RUN: cat %s >> %t.s
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index fe5832b286a2b3..86ed4db05ce05f 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17109,6 +17109,34 @@ It supports AMDGCN GFX6-GFX11.
This section describes general syntax for instructions and operands.
+(amdgpu-async-barrier-replay-warning)=
+
+#### Async Barrier Arrival Warning
+
+The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
+unless XNACK is explicitly disabled. The assembler checks hand-written and
+inline assembly for the same sequence. This check is also omitted when XNACK
+is explicitly disabled.
+
+The assembler warns by default when it cannot recognize the compiler's dummy
+form: `s_mov_b32 exec_lo, 0` followed by
+`global_load_async_to_lds_b8` with a vector address, issued after an arrival
+and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
+not change EXEC, may intervene between the EXEC clear and the load.
+Other load opcodes, scalar-address forms,
+and active loads do not suppress the warning. Recognition stops at labels,
+control-flow instructions, and the end of assembly. Assembler directives
+can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
+block is checked separately. A warning reports missing recognized protection;
+absence of a warning does not prove control-flow safety, correct EXEC
+preservation, or correct async wait counts.
+
+Disable only this check in Clang with
+`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
+`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
+suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
+errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
+
#### Instructions
An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 75318319d88b92..bbf4618825417f 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,12 @@ Makes programs 10x faster by doing Special New Thing.
### Changes to the AMDGPU Backend
+* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
+ assembler warnings for unrecognized sequences in handwritten or inline
+ assembly. Both are omitted when XNACK is explicitly disabled. See
+ {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
+ warning controls.
+
* Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
and `amdgpu.sramecc` module flags.
* `llvm.amdgcn.make.buffer.rsrc` now accepts any integer width for its
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index a8c6d5ef457639..19314b0f240be3 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
#include "Utils/AMDKernelCodeTUtils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
@@ -40,10 +41,12 @@
#include "llvm/Support/AMDGPUObjLinkingInfo.h"
#include "llvm/Support/AMDHSAKernelDescriptor.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
#include <optional>
+#include <utility>
using namespace llvm;
using namespace llvm::AMDGPU;
@@ -51,6 +54,12 @@ using namespace llvm::amdhsa;
namespace {
+static cl::opt<bool> WarnAsyncBarrierReplay(
+ "amdgpu-asm-warn-async-barrier-replay",
+ cl::desc("Warn about async barrier arrivals without a recognized replay "
+ "workaround in assembly"),
+ cl::init(true), cl::Hidden);
+
class AMDGPUAsmParser;
enum RegisterKind {
@@ -1135,6 +1144,12 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
const AMDGPU::GPUKind Gfx;
const AMDGPU::IsaVersion ISA;
+ SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
+ bool AsyncBarrierExecCleared = false;
+
+ void checkAsyncBarrierReplay(const MCInst &Inst);
+ void warnPendingAsyncBarrierArrivals();
+
/// @name Auto-generated Match Functions
/// {
@@ -5839,6 +5854,64 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
return false;
}
+void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
+ for (SMLoc Loc : PendingAsyncBarrierArrivals) {
+ Warning(Loc, "no recognized replay workaround after "
+ "'ds_atomic_async_barrier_arrive_b64' on this GPU");
+ }
+ PendingAsyncBarrierArrivals.clear();
+ AsyncBarrierExecCleared = false;
+}
+
+void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
+ if (!WarnAsyncBarrierReplay ||
+ !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
+ getTargetStreamer().getTargetID()->getXnackSetting() ==
+ AMDGPU::TargetIDSetting::Off)
+ return;
+
+ const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
+ switch (Inst.getOpcode()) {
+ case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
+ PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
+ return;
+ // These instructions change control flow but their MC descriptors do not
+ // carry branch, return, or trap flags.
+ case AMDGPU::S_RFE_B64_gfx1250:
+ case AMDGPU::S_TRAP_gfx12:
+ warnPendingAsyncBarrierArrivals();
+ return;
+ case AMDGPU::S_MOV_B32_gfx12:
+ if (!PendingAsyncBarrierArrivals.empty()) {
+ const MCOperand &Dst = Inst.getOperand(0);
+ const MCOperand &Src = Inst.getOperand(1);
+ AsyncBarrierExecCleared = Dst.isReg() &&
+ mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
+ Src.isImm() && Src.getImm() == 0;
+ }
+ return;
+ case AMDGPU::S_SET_VGPR_MSB_gfx12:
+ // VGPR encoding can insert this between clearing EXEC and the dummy load.
+ AsyncBarrierExecCleared = ExecCleared;
+ return;
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
+ // Recognize only the compiler's EXEC-zero B8 dummy form.
+ if (ExecCleared)
+ PendingAsyncBarrierArrivals.clear();
+ return;
+ }
+
+ if (PendingAsyncBarrierArrivals.empty())
+ return;
+
+ const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
+ // Check only a straight-line sequence. Labels and control-flow changes end
+ // recognition rather than allowing a load on another path to hide a warning.
+ if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
+ (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
+ warnPendingAsyncBarrierArrivals();
+}
+
bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
OperandVector &Operands,
MCStreamer &Out,
@@ -5878,6 +5951,7 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
return true;
}
emitTargetDirective();
+ checkAsyncBarrierReplay(Inst);
Out.emitInstruction(Inst, getSTI());
// Record for kernel prologue checking.
OpcodeStream.push_back(Inst.getOpcode());
@@ -7027,6 +7101,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
}
void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
+ warnPendingAsyncBarrierArrivals();
// Record every parsed label in the timeline so that, at end of file, the
// instructions following a kernel's label can be located regardless of
// whether the .amdhsa_kernel directive came before or after the label.
@@ -7057,6 +7132,7 @@ void AMDGPUAsmParser::checkKernelPrologues() {
}
void AMDGPUAsmParser::onEndOfFile() {
+ warnPendingAsyncBarrierArrivals();
emitTargetDirective();
checkKernelPrologues();
if (InfoData)
@@ -7066,6 +7142,11 @@ void AMDGPUAsmParser::onEndOfFile() {
bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
StringRef IDVal = DirectiveID.getString();
+ // Debug locations do not interrupt the instruction sequence. Other
+ // directives may change sections or emit bytes outside this parser.
+ if (IDVal != ".loc" && IDVal != ".file")
+ warnPendingAsyncBarrierArrivals();
+
if (isHsaAbi(getSTI())) {
if (IDVal == ".amdhsa_kernel")
return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index 313f67ea31e22f..cc8ed4cdd251f3 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -265,6 +265,12 @@ def FeaturePreciseMemory
: SubtargetFeature<"precise-memory", "EnablePreciseMemory",
"true", "Enable precise memory mode">;
+defm AsyncBarrierArriveReplayBug : AMDGPUSubtargetFeature<
+ "async-barrier-arrive-replay-bug",
+ "Enable the async barrier arrival workaround",
+ /*GenPredicate=*/0, /*GenAssemblerPredicate=*/0, [], InlineIgnore
+>;
+
defm SGPRInitBug : AMDGPUSubtargetFeature<"sgpr-init-bug",
"VI SGPR initialization bug requiring a fixed SGPR allocation size",
/*GenPredicate=*/1, /*GenAssemblerPredicate=*/1, [], InlineIgnore
@@ -2493,6 +2499,7 @@ def FeatureISAVersion12 : FeatureSet<
def FeatureISAVersion12_50_Common : FeatureSet<
[FeatureGFX12,
+ FeatureAsyncBarrierArriveReplayBug,
FeatureGFX1250Insts,
FeatureBackOffBarrier,
FeatureRequiresAlignedVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 38604fb4741b36..07829fcc1486cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -345,6 +345,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool isXNACKEnabled() const { return TargetID.isXnackOnOrAny(); }
+ bool needsAsyncBarrierArriveReplayWorkaround() const {
+ // Generic 12.5 code has no explicit XNACK setting; keep the workaround.
+ return hasAsyncBarrierArriveReplayBug() &&
+ TargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Off;
+ }
+
bool hasRelaxedBufferOOBMode() const { return BufferOOBRelaxed; }
bool hasRelaxedTBufferOOBMode() const { return TBufferOOBRelaxed; }
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index cb51eeb8b13c96..ce761f94b97a2f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1859,7 +1859,20 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
// If found any unused scratch SGPR, reserve the register itself for Exec
// copy and there is no need for any spills in that case.
MFI->setSGPRForEXECCopy(UnusedScratchReg);
- MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
+ // Only rewrite EXEC saves and restores. Other uses, such as inline
+ // assembly operands and their callee saves, name fixed registers.
+ const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+ for (MachineOperand &Op :
+ make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
+ MachineInstr &MI = *Op.getParent();
+ if (MI.getOpcode() == LMC.OrSaveExecOpc ||
+ (MI.getOpcode() == LMC.MovOpc &&
+ (MI.getOperand(0).getReg() == LMC.ExecReg ||
+ (MI.getOperand(1).isReg() &&
+ MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+ Op.setReg(UnusedScratchReg);
+ }
+ }
LiveUnits.addReg(UnusedScratchReg);
} else {
// Needs spill.
@@ -1933,19 +1946,23 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo();
bool NeedExecCopyReservedReg = false;
+ bool NeedExecCopyForAsyncBarrier = false;
MachineInstr *ReturnMI = nullptr;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// TODO: Walking through all MBBs here would be a bad heuristic. Better
// handle them elsewhere.
- if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+ if (TII->isWWMRegSpillOpcode(MI.getOpcode())) {
NeedExecCopyReservedReg = true;
- else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
- MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
- MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
- (MFI->isChainFunction() &&
- TII->isChainCallOpcode(MI.getOpcode()))) {
+ } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
+ ST.needsAsyncBarrierArriveReplayWorkaround()) {
+ NeedExecCopyForAsyncBarrier = true;
+ } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
+ MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
+ MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
+ (MFI->isChainFunction() &&
+ TII->isChainCallOpcode(MI.getOpcode()))) {
// We expect all return to be the same size.
assert(!ReturnMI ||
(count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
@@ -2013,7 +2030,10 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
if (!ST.hasGFX90AInsts())
SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
- determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
+ // Whole-wave spills use the original EXEC mask from the first argument, but
+ // the barrier workaround needs separate storage for the current EXEC mask.
+ determinePrologEpilogSGPRSaves(
+ MF, SavedVGPRs, NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
// The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
// allow the default insertion to handle them.
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bd2815d0491b5b..9fa338a3f1826f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -3497,6 +3498,43 @@ bool SIInsertWaitcnts::run() {
bool Modified = false;
+ if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
+ // Collect arrivals before inserting new definitions of ASYNCcnt.
+ SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+ for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+ if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+ AsyncBarrierArrivals.push_back(&MI);
+ }
+
+ if (!AsyncBarrierArrivals.empty()) {
+ const Register SavedExec = MFI->getSGPRForEXECCopy();
+ if (!SavedExec) {
+ report_fatal_error(
+ "missing EXEC copy SGPR for async barrier workaround");
+ }
+
+ for (MachineInstr *MI : AsyncBarrierArrivals) {
+ MachineBasicBlock &MBB = *MI->getParent();
+ const DebugLoc &DL = MI->getDebugLoc();
+ // Insert after scheduling so the dummy stays after its arrival.
+ auto InsertPt = std::next(MI->getIterator());
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+ .addReg(AMDGPU::EXEC_LO);
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+ .addImm(0);
+ // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+ .addImm(0)
+ .addImm(0);
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+ .addReg(SavedExec);
+ Modified = true;
+ }
+ }
+ }
+
MachineBasicBlock &EntryBB = MF.front();
if (!MFI->isEntryFunction() &&
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index d0a57e163d29a0..f48e2194242a42 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -619,9 +619,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
- } else {
- // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
- // spills/copies. Reset the SGPR reserved for EXEC copy.
+ } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
+ none_of(MRI.def_instructions(AMDGPU::ASYNCcnt),
+ [](const MachineInstr &MI) {
+ return MI.getOpcode() ==
+ AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
+ })) {
+ // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
+ // needs the reserved register for EXEC copy.
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/238539
More information about the cfe-commits
mailing list