[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)
Yaxun Liu via cfe-commits
cfe-commits at lists.llvm.org
Tue Nov 3 13:47:55 PST 2026
https://github.com/yxsamliu created https://github.com/llvm/llvm-project/pull/238539
Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.
Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.
>From f8361a7e7d813edf75bbaf1677b22848bdd7f62d Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 3 Nov 2026 12:01:44 -0500
Subject: [PATCH] [AMDGPU] Add an async barrier arrival workaround
Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.
Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.
---
.../AMDGPU/async-barrier-replay-warning.c | 20 +
.../amdgpu-async-barrier-replay-warning.s | 15 +
llvm/docs/AMDGPUUsage.md | 28 +
llvm/docs/ReleaseNotes.md | 6 +
.../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 81 +++
llvm/lib/Target/AMDGPU/GCNFeatures.td | 7 +
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 6 +
llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 36 +-
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 38 ++
llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp | 11 +-
.../async-barrier-arrive-replay-vgpr-msb.mir | 32 ++
.../async-barrier-arrive-replay-whole-wave.ll | 494 ++++++++++++++++++
.../AMDGPU/async-barrier-arrive-replay.ll | 46 ++
.../AMDGPU/async-barrier-arrive-replay.mir | 269 ++++++++++
.../AMDGPU/async-barrier-replay-inline-asm.ll | 24 +
.../AMDGPU/ds-atomic-barrier-convergent.ll | 4 +
...dgcn.ds.atomic.async.barrier.arrive.b64.ll | 12 +
.../async-barrier-replay-warning-vgpr-msb.s | 38 ++
.../async-barrier-replay-warning-xnack.s | 27 +
.../MC/AMDGPU/async-barrier-replay-warning.s | 187 +++++++
20 files changed, 1370 insertions(+), 11 deletions(-)
create mode 100644 clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
create mode 100644 clang/test/Driver/amdgpu-async-barrier-replay-warning.s
create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
new file mode 100644
index 00000000000000..b0b57f42c26bd0
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
@@ -0,0 +1,20 @@
+// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
+
+void unprotected(void) {
+ __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
+// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+}
+
+void protected(void) {
+ __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
+ "s_mov_b32 s7, exec_lo\n"
+ "s_mov_b32 exec_lo, 0\n"
+ "global_load_async_to_lds_b8 v0, v[0:1], off\n"
+ "s_mov_b32 exec_lo, s7"
+ ::: "v0", "v1", "s7", "memory");
+}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..3a1549c3b325e3
--- /dev/null
+++ b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
@@ -0,0 +1,15 @@
+// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
+// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
+// RUN: cat %s >> %t.s
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index fe5832b286a2b3..86ed4db05ce05f 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17109,6 +17109,34 @@ It supports AMDGCN GFX6-GFX11.
This section describes general syntax for instructions and operands.
+(amdgpu-async-barrier-replay-warning)=
+
+#### Async Barrier Arrival Warning
+
+The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
+unless XNACK is explicitly disabled. The assembler checks hand-written and
+inline assembly for the same sequence. This check is also omitted when XNACK
+is explicitly disabled.
+
+The assembler warns by default when it cannot recognize the compiler's dummy
+form: `s_mov_b32 exec_lo, 0` followed by
+`global_load_async_to_lds_b8` with a vector address, issued after an arrival
+and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
+not change EXEC, may intervene between the EXEC clear and the load.
+Other load opcodes, scalar-address forms,
+and active loads do not suppress the warning. Recognition stops at labels,
+control-flow instructions, and the end of assembly. Assembler directives
+can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
+block is checked separately. A warning reports missing recognized protection;
+absence of a warning does not prove control-flow safety, correct EXEC
+preservation, or correct async wait counts.
+
+Disable only this check in Clang with
+`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
+`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
+suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
+errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
+
#### Instructions
An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 75318319d88b92..bbf4618825417f 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,12 @@ Makes programs 10x faster by doing Special New Thing.
### Changes to the AMDGPU Backend
+* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
+ assembler warnings for unrecognized sequences in handwritten or inline
+ assembly. Both are omitted when XNACK is explicitly disabled. See
+ {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
+ warning controls.
+
* Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
and `amdgpu.sramecc` module flags.
* `llvm.amdgcn.make.buffer.rsrc` now accepts any integer width for its
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index a8c6d5ef457639..19314b0f240be3 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
#include "Utils/AMDKernelCodeTUtils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
@@ -40,10 +41,12 @@
#include "llvm/Support/AMDGPUObjLinkingInfo.h"
#include "llvm/Support/AMDHSAKernelDescriptor.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
#include <optional>
+#include <utility>
using namespace llvm;
using namespace llvm::AMDGPU;
@@ -51,6 +54,12 @@ using namespace llvm::amdhsa;
namespace {
+static cl::opt<bool> WarnAsyncBarrierReplay(
+ "amdgpu-asm-warn-async-barrier-replay",
+ cl::desc("Warn about async barrier arrivals without a recognized replay "
+ "workaround in assembly"),
+ cl::init(true), cl::Hidden);
+
class AMDGPUAsmParser;
enum RegisterKind {
@@ -1135,6 +1144,12 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
const AMDGPU::GPUKind Gfx;
const AMDGPU::IsaVersion ISA;
+ SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
+ bool AsyncBarrierExecCleared = false;
+
+ void checkAsyncBarrierReplay(const MCInst &Inst);
+ void warnPendingAsyncBarrierArrivals();
+
/// @name Auto-generated Match Functions
/// {
@@ -5839,6 +5854,64 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
return false;
}
+void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
+ for (SMLoc Loc : PendingAsyncBarrierArrivals) {
+ Warning(Loc, "no recognized replay workaround after "
+ "'ds_atomic_async_barrier_arrive_b64' on this GPU");
+ }
+ PendingAsyncBarrierArrivals.clear();
+ AsyncBarrierExecCleared = false;
+}
+
+void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
+ if (!WarnAsyncBarrierReplay ||
+ !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
+ getTargetStreamer().getTargetID()->getXnackSetting() ==
+ AMDGPU::TargetIDSetting::Off)
+ return;
+
+ const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
+ switch (Inst.getOpcode()) {
+ case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
+ PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
+ return;
+ // These instructions change control flow but their MC descriptors do not
+ // carry branch, return, or trap flags.
+ case AMDGPU::S_RFE_B64_gfx1250:
+ case AMDGPU::S_TRAP_gfx12:
+ warnPendingAsyncBarrierArrivals();
+ return;
+ case AMDGPU::S_MOV_B32_gfx12:
+ if (!PendingAsyncBarrierArrivals.empty()) {
+ const MCOperand &Dst = Inst.getOperand(0);
+ const MCOperand &Src = Inst.getOperand(1);
+ AsyncBarrierExecCleared = Dst.isReg() &&
+ mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
+ Src.isImm() && Src.getImm() == 0;
+ }
+ return;
+ case AMDGPU::S_SET_VGPR_MSB_gfx12:
+ // VGPR encoding can insert this between clearing EXEC and the dummy load.
+ AsyncBarrierExecCleared = ExecCleared;
+ return;
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
+ // Recognize only the compiler's EXEC-zero B8 dummy form.
+ if (ExecCleared)
+ PendingAsyncBarrierArrivals.clear();
+ return;
+ }
+
+ if (PendingAsyncBarrierArrivals.empty())
+ return;
+
+ const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
+ // Check only a straight-line sequence. Labels and control-flow changes end
+ // recognition rather than allowing a load on another path to hide a warning.
+ if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
+ (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
+ warnPendingAsyncBarrierArrivals();
+}
+
bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
OperandVector &Operands,
MCStreamer &Out,
@@ -5878,6 +5951,7 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
return true;
}
emitTargetDirective();
+ checkAsyncBarrierReplay(Inst);
Out.emitInstruction(Inst, getSTI());
// Record for kernel prologue checking.
OpcodeStream.push_back(Inst.getOpcode());
@@ -7027,6 +7101,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
}
void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
+ warnPendingAsyncBarrierArrivals();
// Record every parsed label in the timeline so that, at end of file, the
// instructions following a kernel's label can be located regardless of
// whether the .amdhsa_kernel directive came before or after the label.
@@ -7057,6 +7132,7 @@ void AMDGPUAsmParser::checkKernelPrologues() {
}
void AMDGPUAsmParser::onEndOfFile() {
+ warnPendingAsyncBarrierArrivals();
emitTargetDirective();
checkKernelPrologues();
if (InfoData)
@@ -7066,6 +7142,11 @@ void AMDGPUAsmParser::onEndOfFile() {
bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
StringRef IDVal = DirectiveID.getString();
+ // Debug locations do not interrupt the instruction sequence. Other
+ // directives may change sections or emit bytes outside this parser.
+ if (IDVal != ".loc" && IDVal != ".file")
+ warnPendingAsyncBarrierArrivals();
+
if (isHsaAbi(getSTI())) {
if (IDVal == ".amdhsa_kernel")
return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index 313f67ea31e22f..cc8ed4cdd251f3 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -265,6 +265,12 @@ def FeaturePreciseMemory
: SubtargetFeature<"precise-memory", "EnablePreciseMemory",
"true", "Enable precise memory mode">;
+defm AsyncBarrierArriveReplayBug : AMDGPUSubtargetFeature<
+ "async-barrier-arrive-replay-bug",
+ "Enable the async barrier arrival workaround",
+ /*GenPredicate=*/0, /*GenAssemblerPredicate=*/0, [], InlineIgnore
+>;
+
defm SGPRInitBug : AMDGPUSubtargetFeature<"sgpr-init-bug",
"VI SGPR initialization bug requiring a fixed SGPR allocation size",
/*GenPredicate=*/1, /*GenAssemblerPredicate=*/1, [], InlineIgnore
@@ -2493,6 +2499,7 @@ def FeatureISAVersion12 : FeatureSet<
def FeatureISAVersion12_50_Common : FeatureSet<
[FeatureGFX12,
+ FeatureAsyncBarrierArriveReplayBug,
FeatureGFX1250Insts,
FeatureBackOffBarrier,
FeatureRequiresAlignedVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 38604fb4741b36..07829fcc1486cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -345,6 +345,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool isXNACKEnabled() const { return TargetID.isXnackOnOrAny(); }
+ bool needsAsyncBarrierArriveReplayWorkaround() const {
+ // Generic 12.5 code has no explicit XNACK setting; keep the workaround.
+ return hasAsyncBarrierArriveReplayBug() &&
+ TargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Off;
+ }
+
bool hasRelaxedBufferOOBMode() const { return BufferOOBRelaxed; }
bool hasRelaxedTBufferOOBMode() const { return TBufferOOBRelaxed; }
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index cb51eeb8b13c96..ce761f94b97a2f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1859,7 +1859,20 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
// If found any unused scratch SGPR, reserve the register itself for Exec
// copy and there is no need for any spills in that case.
MFI->setSGPRForEXECCopy(UnusedScratchReg);
- MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
+ // Only rewrite EXEC saves and restores. Other uses, such as inline
+ // assembly operands and their callee saves, name fixed registers.
+ const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+ for (MachineOperand &Op :
+ make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
+ MachineInstr &MI = *Op.getParent();
+ if (MI.getOpcode() == LMC.OrSaveExecOpc ||
+ (MI.getOpcode() == LMC.MovOpc &&
+ (MI.getOperand(0).getReg() == LMC.ExecReg ||
+ (MI.getOperand(1).isReg() &&
+ MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+ Op.setReg(UnusedScratchReg);
+ }
+ }
LiveUnits.addReg(UnusedScratchReg);
} else {
// Needs spill.
@@ -1933,19 +1946,23 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo();
bool NeedExecCopyReservedReg = false;
+ bool NeedExecCopyForAsyncBarrier = false;
MachineInstr *ReturnMI = nullptr;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// TODO: Walking through all MBBs here would be a bad heuristic. Better
// handle them elsewhere.
- if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+ if (TII->isWWMRegSpillOpcode(MI.getOpcode())) {
NeedExecCopyReservedReg = true;
- else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
- MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
- MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
- (MFI->isChainFunction() &&
- TII->isChainCallOpcode(MI.getOpcode()))) {
+ } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
+ ST.needsAsyncBarrierArriveReplayWorkaround()) {
+ NeedExecCopyForAsyncBarrier = true;
+ } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
+ MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
+ MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
+ (MFI->isChainFunction() &&
+ TII->isChainCallOpcode(MI.getOpcode()))) {
// We expect all return to be the same size.
assert(!ReturnMI ||
(count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
@@ -2013,7 +2030,10 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
if (!ST.hasGFX90AInsts())
SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
- determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
+ // Whole-wave spills use the original EXEC mask from the first argument, but
+ // the barrier workaround needs separate storage for the current EXEC mask.
+ determinePrologEpilogSGPRSaves(
+ MF, SavedVGPRs, NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
// The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
// allow the default insertion to handle them.
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bd2815d0491b5b..9fa338a3f1826f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -3497,6 +3498,43 @@ bool SIInsertWaitcnts::run() {
bool Modified = false;
+ if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
+ // Collect arrivals before inserting new definitions of ASYNCcnt.
+ SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+ for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+ if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+ AsyncBarrierArrivals.push_back(&MI);
+ }
+
+ if (!AsyncBarrierArrivals.empty()) {
+ const Register SavedExec = MFI->getSGPRForEXECCopy();
+ if (!SavedExec) {
+ report_fatal_error(
+ "missing EXEC copy SGPR for async barrier workaround");
+ }
+
+ for (MachineInstr *MI : AsyncBarrierArrivals) {
+ MachineBasicBlock &MBB = *MI->getParent();
+ const DebugLoc &DL = MI->getDebugLoc();
+ // Insert after scheduling so the dummy stays after its arrival.
+ auto InsertPt = std::next(MI->getIterator());
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+ .addReg(AMDGPU::EXEC_LO);
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+ .addImm(0);
+ // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+ .addImm(0)
+ .addImm(0);
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+ .addReg(SavedExec);
+ Modified = true;
+ }
+ }
+ }
+
MachineBasicBlock &EntryBB = MF.front();
if (!MFI->isEntryFunction() &&
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index d0a57e163d29a0..f48e2194242a42 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -619,9 +619,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
- } else {
- // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
- // spills/copies. Reset the SGPR reserved for EXEC copy.
+ } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
+ none_of(MRI.def_instructions(AMDGPU::ASYNCcnt),
+ [](const MachineInstr &MI) {
+ return MI.getOpcode() ==
+ AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
+ })) {
+ // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
+ // needs the reserved register for EXEC copy.
FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
new file mode 100644
index 00000000000000..155f0d3b296a3a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
@@ -0,0 +1,32 @@
+# NOTE: Do not autogenerate. Check the bank switch inside the workaround and assemble compiler output with fatal warnings.
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
+
+--- |
+ define amdgpu_cs void @high_vgpr_arrive() { ret void }
+...
+
+---
+name: high_vgpr_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ bb.0:
+ liveins: $vgpr256
+ ; The arrival needs a high VGPR, but the dummy uses v[0:1]. The late
+ ; encoding pass resets the bank after EXEC is cleared.
+ ; CHECK-LABEL: high_vgpr_arrive:
+ ; CHECK: s_set_vgpr_msb 1
+ ; CHECK-NEXT: ds_atomic_async_barrier_arrive_b64 v0 /*v256*/
+ ; CHECK: s_mov_b32 s105, exec_lo
+ ; CHECK-NEXT: s_mov_b32 exec_lo, 0
+ ; CHECK-NEXT: s_set_vgpr_msb 0x100
+ ; CHECK-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+ ; CHECK-NEXT: s_mov_b32 exec_lo, s105
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr256, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
new file mode 100644
index 00000000000000..5998e619fbcab7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
@@ -0,0 +1,494 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+
+; The caller initializes the LDS barrier at offset zero. Only the active-mask
+; argument is used, since ordinary arguments are poison in inactive lanes.
+; gfx1250 and gfx1251 do not support wave64.
+
+; The barrier must use a separate scratch SGPR from the saved incoming EXEC,
+; even when the active argument has no explicit uses.
+define amdgpu_gfx_whole_wave void @whole_wave_barrier(i1 %active) {
+; ON-LABEL: whole_wave_barrier:
+; ON: ; %bb.0:
+; ON-NEXT: s_wait_loadcnt_dscnt 0x0
+; ON-NEXT: s_wait_kmcnt 0x0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, -1
+; ON-NEXT: v_mov_b32_e32 v0, 0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: s_mov_b32 s1, exec_lo
+; ON-NEXT: s_mov_b32 exec_lo, 0
+; ON-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT: s_mov_b32 exec_lo, s1
+; ON-NEXT: s_xor_b32 exec_lo, s0, -1
+; ON-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_wait_loadcnt 0x0
+; ON-NEXT: s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_barrier:
+; OFF: ; %bb.0:
+; OFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT: s_wait_kmcnt 0x0
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT: s_mov_b32 exec_lo, -1
+; OFF-NEXT: v_mov_b32_e32 v0, 0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: s_wait_loadcnt 0x0
+; OFF-NEXT: s_set_pc_i64 s[30:31]
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ ret void
+}
+
+; Moving the EXEC scratch must preserve fixed-register assembly operands and the
+; corresponding callee saves, in both whole-wave and ordinary functions.
+define amdgpu_gfx_whole_wave i32 @whole_wave_clobber_exec_scratch(i1 %active) {
+; ON-LABEL: whole_wave_clobber_exec_scratch:
+; ON: ; %bb.0:
+; ON-NEXT: s_wait_loadcnt_dscnt 0x0
+; ON-NEXT: s_wait_kmcnt 0x0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT: scratch_store_b32 off, v2, s32 nv
+; ON-NEXT: ; meta instruction
+; ON-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; ON-NEXT: ; meta instruction
+; ON-NEXT: scratch_store_b32 off, v1, s32 offset:8 nv
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, -1
+; ON-NEXT: v_writelane_b32 v2, s105, 0
+; ON-NEXT: ;;#ASMSTART
+; ON-NEXT: s_mov_b32 s105, 0
+; ON-NEXT: ;;#ASMEND
+; ON-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT: v_readlane_b32 s105, v2, 0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: s_mov_b32 s1, exec_lo
+; ON-NEXT: s_mov_b32 exec_lo, 0
+; ON-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT: s_mov_b32 exec_lo, s1
+; ON-NEXT: s_xor_b32 exec_lo, s0, -1
+; ON-NEXT: s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT: scratch_load_b32 v2, off, s32 nv
+; ON-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; ON-NEXT: scratch_load_b32 v1, off, s32 offset:8 nv
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_wait_loadcnt 0x0
+; ON-NEXT: s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_clobber_exec_scratch:
+; OFF: ; %bb.0:
+; OFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT: s_wait_kmcnt 0x0
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT: scratch_store_b32 off, v2, s32 nv
+; OFF-NEXT: ; meta instruction
+; OFF-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; OFF-NEXT: ; meta instruction
+; OFF-NEXT: scratch_store_b32 off, v1, s32 offset:8 nv
+; OFF-NEXT: s_mov_b32 exec_lo, -1
+; OFF-NEXT: v_writelane_b32 v2, s105, 0
+; OFF-NEXT: ;;#ASMSTART
+; OFF-NEXT: s_mov_b32 s105, 0
+; OFF-NEXT: ;;#ASMEND
+; OFF-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT: v_readlane_b32 s105, v2, 0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT: s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT: scratch_load_b32 v2, off, s32 nv
+; OFF-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; OFF-NEXT: scratch_load_b32 v1, off, s32 offset:8 nv
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: s_wait_loadcnt 0x0
+; OFF-NEXT: s_set_pc_i64 s[30:31]
+ %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ ret i32 %value
+}
+
+define i32 @callable_clobber_exec_scratch() {
+; ON-LABEL: callable_clobber_exec_scratch:
+; ON: ; %bb.0:
+; ON-NEXT: s_wait_loadcnt_dscnt 0x0
+; ON-NEXT: s_wait_kmcnt 0x0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: v_writelane_b32 v2, s105, 0
+; ON-NEXT: ;;#ASMSTART
+; ON-NEXT: s_mov_b32 s105, 0
+; ON-NEXT: ;;#ASMEND
+; ON-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT: v_readlane_b32 s105, v2, 0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: s_mov_b32 s0, exec_lo
+; ON-NEXT: s_mov_b32 exec_lo, 0
+; ON-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_wait_loadcnt 0x0
+; ON-NEXT: s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: callable_clobber_exec_scratch:
+; OFF: ; %bb.0:
+; OFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT: s_wait_kmcnt 0x0
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: v_writelane_b32 v2, s105, 0
+; OFF-NEXT: ;;#ASMSTART
+; OFF-NEXT: s_mov_b32 s105, 0
+; OFF-NEXT: ;;#ASMEND
+; OFF-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT: v_readlane_b32 s105, v2, 0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: s_wait_loadcnt 0x0
+; OFF-NEXT: s_set_pc_i64 s[30:31]
+ %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ ret i32 %value
+}
+
+; Keep the active argument live across the barrier and exercise SGPR and VGPR
+; callee saves as well as the reserved scratch register.
+define amdgpu_gfx_whole_wave i32 @whole_wave_active(i1 %active) {
+; SDAG-LABEL: whole_wave_active:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: s_xor_saveexec_b32 s0, -1
+; SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; SDAG-NEXT: scratch_store_b32 off, v1, s32 nv
+; SDAG-NEXT: ; meta instruction
+; SDAG-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: s_mov_b32 exec_lo, -1
+; SDAG-NEXT: scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAG-NEXT: v_writelane_b32 v1, s68, 0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: ;;#ASMSTART
+; SDAG-NEXT: ; clobber CSR
+; SDAG-NEXT: ;;#ASMEND
+; SDAG-NEXT: s_and_b32 s1, s0, exec_lo
+; SDAG-NEXT: s_cselect_b32 s1, 42, 7
+; SDAG-NEXT: v_readlane_b32 s68, v1, 0
+; SDAG-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; SDAG-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT: s_mov_b32 s2, exec_lo
+; SDAG-NEXT: s_mov_b32 exec_lo, 0
+; SDAG-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; SDAG-NEXT: s_mov_b32 exec_lo, s2
+; SDAG-NEXT: scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: s_xor_b32 exec_lo, s0, -1
+; SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; SDAG-NEXT: scratch_load_b32 v1, off, s32 nv
+; SDAG-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: s_mov_b32 exec_lo, s0
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; SDAGOFF-LABEL: whole_wave_active:
+; SDAGOFF: ; %bb.0:
+; SDAGOFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAGOFF-NEXT: s_wait_kmcnt 0x0
+; SDAGOFF-NEXT: s_xor_saveexec_b32 s0, -1
+; SDAGOFF-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; SDAGOFF-NEXT: scratch_store_b32 off, v1, s32 nv
+; SDAGOFF-NEXT: ; meta instruction
+; SDAGOFF-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAGOFF-NEXT: s_mov_b32 exec_lo, -1
+; SDAGOFF-NEXT: scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAGOFF-NEXT: v_writelane_b32 v1, s68, 0
+; SDAGOFF-NEXT: v_mov_b32_e32 v0, 0
+; SDAGOFF-NEXT: ;;#ASMSTART
+; SDAGOFF-NEXT: ; clobber CSR
+; SDAGOFF-NEXT: ;;#ASMEND
+; SDAGOFF-NEXT: s_and_b32 s1, s0, exec_lo
+; SDAGOFF-NEXT: s_cselect_b32 s1, 42, 7
+; SDAGOFF-NEXT: v_readlane_b32 s68, v1, 0
+; SDAGOFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; SDAGOFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT: scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAGOFF-NEXT: v_mov_b32_e32 v0, s1
+; SDAGOFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; SDAGOFF-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; SDAGOFF-NEXT: scratch_load_b32 v1, off, s32 nv
+; SDAGOFF-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAGOFF-NEXT: s_mov_b32 exec_lo, s0
+; SDAGOFF-NEXT: s_wait_loadcnt 0x0
+; SDAGOFF-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: whole_wave_active:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: s_xor_saveexec_b32 s0, -1
+; GISEL-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GISEL-NEXT: scratch_store_b32 off, v1, s32 nv
+; GISEL-NEXT: ; meta instruction
+; GISEL-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_mov_b32 exec_lo, -1
+; GISEL-NEXT: scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISEL-NEXT: v_writelane_b32 v1, s68, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: ;;#ASMSTART
+; GISEL-NEXT: ; clobber CSR
+; GISEL-NEXT: ;;#ASMEND
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GISEL-NEXT: v_readlane_b32 s68, v1, 0
+; GISEL-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; GISEL-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GISEL-NEXT: s_mov_b32 exec_lo, 0
+; GISEL-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; GISEL-NEXT: s_mov_b32 exec_lo, s1
+; GISEL-NEXT: scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 7, 42, s0
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_xor_b32 exec_lo, s0, -1
+; GISEL-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GISEL-NEXT: scratch_load_b32 v1, off, s32 nv
+; GISEL-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISELOFF-LABEL: whole_wave_active:
+; GISELOFF: ; %bb.0:
+; GISELOFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISELOFF-NEXT: s_wait_kmcnt 0x0
+; GISELOFF-NEXT: s_xor_saveexec_b32 s0, -1
+; GISELOFF-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GISELOFF-NEXT: scratch_store_b32 off, v1, s32 nv
+; GISELOFF-NEXT: ; meta instruction
+; GISELOFF-NEXT: scratch_store_b32 off, v0, s32 offset:4 nv
+; GISELOFF-NEXT: s_mov_b32 exec_lo, -1
+; GISELOFF-NEXT: scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISELOFF-NEXT: v_writelane_b32 v1, s68, 0
+; GISELOFF-NEXT: v_mov_b32_e32 v0, 0
+; GISELOFF-NEXT: ;;#ASMSTART
+; GISELOFF-NEXT: ; clobber CSR
+; GISELOFF-NEXT: ;;#ASMEND
+; GISELOFF-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GISELOFF-NEXT: v_readlane_b32 s68, v1, 0
+; GISELOFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; GISELOFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT: scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISELOFF-NEXT: v_cndmask_b32_e64 v0, 7, 42, s0
+; GISELOFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; GISELOFF-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GISELOFF-NEXT: scratch_load_b32 v1, off, s32 nv
+; GISELOFF-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; GISELOFF-NEXT: s_mov_b32 exec_lo, s0
+; GISELOFF-NEXT: s_wait_loadcnt 0x0
+; GISELOFF-NEXT: s_set_pc_i64 s[30:31]
+ call void asm sideeffect "; clobber CSR", "~{v40},~{s68}"()
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ %result = select i1 %active, i32 42, i32 7
+ ret i32 %result
+}
+
+; A divergent region has a current EXEC mask distinct from the original
+; active argument. Preserve both through the workaround.
+define amdgpu_gfx_whole_wave i32 @whole_wave_divergent(i1 %active) {
+; ON-LABEL: whole_wave_divergent:
+; ON: ; %bb.0:
+; ON-NEXT: s_wait_loadcnt_dscnt 0x0
+; ON-NEXT: s_wait_kmcnt 0x0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, -1
+; ON-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
+; ON-NEXT: s_mov_b32 s1, exec_lo
+; ON-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; ON-NEXT: v_and_b32_e32 v0, 1, v0
+; ON-NEXT: v_cmpx_eq_u32_e32 0, v0
+; ON-NEXT: s_cbranch_execz .LBB4_2
+; ON-NEXT: ; %bb.1: ; %arrive
+; ON-NEXT: v_mov_b32_e32 v0, 0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: s_mov_b32 s2, exec_lo
+; ON-NEXT: s_mov_b32 exec_lo, 0
+; ON-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT: s_mov_b32 exec_lo, s2
+; ON-NEXT: .LBB4_2: ; %exit
+; ON-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; ON-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; ON-NEXT: v_cndmask_b32_e64 v0, 7, 42, s0
+; ON-NEXT: s_xor_b32 exec_lo, s0, -1
+; ON-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_wait_loadcnt 0x0
+; ON-NEXT: s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_divergent:
+; OFF: ; %bb.0:
+; OFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT: s_wait_kmcnt 0x0
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT: s_mov_b32 exec_lo, -1
+; OFF-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
+; OFF-NEXT: s_mov_b32 s1, exec_lo
+; OFF-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; OFF-NEXT: v_and_b32_e32 v0, 1, v0
+; OFF-NEXT: v_cmpx_eq_u32_e32 0, v0
+; OFF-NEXT: ; %bb.1: ; %arrive
+; OFF-NEXT: v_mov_b32_e32 v0, 0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ; %bb.2: ; %exit
+; OFF-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; OFF-NEXT: v_cndmask_b32_e64 v0, 7, 42, s0
+; OFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: s_wait_loadcnt 0x0
+; OFF-NEXT: s_set_pc_i64 s[30:31]
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %parity = and i32 %lane, 1
+ %cond = icmp eq i32 %parity, 0
+ br i1 %cond, label %arrive, label %exit
+
+arrive:
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ br label %exit
+
+exit:
+ %result = select i1 %active, i32 42, i32 7
+ ret i32 %result
+}
+
+; Keep both dummy-load address registers live across the workaround.
+define amdgpu_gfx_whole_wave void @whole_wave_live_vgprs(i1 %active) {
+; ON-LABEL: whole_wave_live_vgprs:
+; ON: ; %bb.0:
+; ON-NEXT: s_wait_loadcnt_dscnt 0x0
+; ON-NEXT: s_wait_kmcnt 0x0
+; ON-NEXT: s_xor_saveexec_b32 s0, -1
+; ON-NEXT: s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT: scratch_store_b32 off, v0, s32 nv
+; ON-NEXT: ; meta instruction
+; ON-NEXT: scratch_store_b32 off, v1, s32 offset:4 nv
+; ON-NEXT: ; meta instruction
+; ON-NEXT: scratch_store_b32 off, v2, s32 offset:8 nv
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, -1
+; ON-NEXT: v_mov_b32_e32 v2, 0
+; ON-NEXT: ;;#ASMSTART
+; ON-NEXT: v_mov_b32_e32 v0, 0x7b
+; ON-NEXT: v_mov_b32_e32 v1, 0x1c8
+; ON-NEXT: ;;#ASMEND
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: ds_atomic_async_barrier_arrive_b64 v2
+; ON-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT: s_mov_b32 s1, exec_lo
+; ON-NEXT: s_mov_b32 exec_lo, 0
+; ON-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT: s_mov_b32 exec_lo, s1
+; ON-NEXT: ;;#ASMSTART
+; ON-NEXT: ; use v[0:1]
+; ON-NEXT: ;;#ASMEND
+; ON-NEXT: s_xor_b32 exec_lo, s0, -1
+; ON-NEXT: s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT: scratch_load_b32 v0, off, s32 nv
+; ON-NEXT: scratch_load_b32 v1, off, s32 offset:4 nv
+; ON-NEXT: scratch_load_b32 v2, off, s32 offset:8 nv
+; ON-NEXT: s_wait_xcnt 0x0
+; ON-NEXT: s_mov_b32 exec_lo, s0
+; ON-NEXT: s_wait_loadcnt 0x0
+; ON-NEXT: s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_live_vgprs:
+; OFF: ; %bb.0:
+; OFF-NEXT: s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT: s_wait_kmcnt 0x0
+; OFF-NEXT: s_xor_saveexec_b32 s0, -1
+; OFF-NEXT: s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT: scratch_store_b32 off, v0, s32 nv
+; OFF-NEXT: ; meta instruction
+; OFF-NEXT: scratch_store_b32 off, v1, s32 offset:4 nv
+; OFF-NEXT: ; meta instruction
+; OFF-NEXT: scratch_store_b32 off, v2, s32 offset:8 nv
+; OFF-NEXT: s_mov_b32 exec_lo, -1
+; OFF-NEXT: v_mov_b32_e32 v2, 0
+; OFF-NEXT: ;;#ASMSTART
+; OFF-NEXT: v_mov_b32_e32 v0, 0x7b
+; OFF-NEXT: v_mov_b32_e32 v1, 0x1c8
+; OFF-NEXT: ;;#ASMEND
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ds_atomic_async_barrier_arrive_b64 v2
+; OFF-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT: ;;#ASMSTART
+; OFF-NEXT: ; use v[0:1]
+; OFF-NEXT: ;;#ASMEND
+; OFF-NEXT: s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT: s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT: scratch_load_b32 v0, off, s32 nv
+; OFF-NEXT: scratch_load_b32 v1, off, s32 offset:4 nv
+; OFF-NEXT: scratch_load_b32 v2, off, s32 offset:8 nv
+; OFF-NEXT: s_mov_b32 exec_lo, s0
+; OFF-NEXT: s_wait_loadcnt 0x0
+; OFF-NEXT: s_set_pc_i64 s[30:31]
+ %values = call <2 x i32> asm sideeffect "v_mov_b32 v0, 123\0Av_mov_b32 v1, 456", "={v[0:1]}"()
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+ call void asm sideeffect "; use $0", "{v[0:1]}"(<2 x i32> %values)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
new file mode 100644
index 00000000000000..6158c7ec97427e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
@@ -0,0 +1,46 @@
+; NOTE: Do not autogenerate. Check workaround ordering on affected targets and absence on unaffected targets.
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu13.10-amd-amdhsa -mattr=+lds-barrier-arrive-atomic < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.5-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-async-barrier-arrive-replay-bug < %s | FileCheck --check-prefix=UNAFFECTED %s
+
+; Check an LDS barrier arrival followed by a global load.
+ at bar = internal addrspace(3) global i64 undef, align 8
+
+; AFFECTED-LABEL: replay_window:
+; AFFECTED: ds_atomic_async_barrier_arrive_b64
+; AFFECTED-NOT: global_load_b32
+; AFFECTED: s_mov_b32 [[SAVED:s[0-9]+]], exec_lo
+; AFFECTED-NEXT: s_mov_b32 exec_lo, 0
+; AFFECTED-NEXT: global_load_async_to_lds_b8
+; AFFECTED-NEXT: s_mov_b32 exec_lo, [[SAVED]]
+; AFFECTED: global_load_b32
+;
+; UNAFFECTED-LABEL: replay_window:
+; UNAFFECTED: ds_atomic_async_barrier_arrive_b64
+; UNAFFECTED-NOT: global_load_async_to_lds
+; UNAFFECTED: global_load_b32
+define amdgpu_kernel void @replay_window(ptr addrspace(1) %src,
+ ptr addrspace(1) %dst) {
+entry:
+ store i64 0, ptr addrspace(3) @bar, align 8
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) @bar)
+ %value = load volatile i32, ptr addrspace(1) %src, align 4
+ store volatile i32 %value, ptr addrspace(1) %dst, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
new file mode 100644
index 00000000000000..09eb65e1ac5cd8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
@@ -0,0 +1,269 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+
+--- |
+ define amdgpu_kernel void @live_scc() { ret void }
+ define amdgpu_kernel void @empty_exec() { ret void }
+ define amdgpu_kernel void @no_arrive() { ret void }
+ define amdgpu_kernel void @multiple_arrivals() { ret void }
+ define amdgpu_kernel void @partial_wait() { ret void }
+ define amdgpu_kernel void @expert_partial_wait() #0 { ret void }
+ attributes #0 = { "amdgpu-expert-scheduling-mode"="true" }
+...
+
+---
+name: live_scc
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; ON-LABEL: name: live_scc
+ ; ON: liveins: $sgpr0, $vgpr0
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: live_scc
+ ; OFF: liveins: $sgpr0, $vgpr0
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+ ; OFF-NEXT: S_ENDPGM 0
+ S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: empty_exec
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ bb.0:
+ liveins: $vgpr0
+ ; ON-LABEL: name: empty_exec
+ ; ON: liveins: $vgpr0
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: $exec_lo = S_MOV_B32 -1
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: empty_exec
+ ; OFF: liveins: $vgpr0
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: $exec_lo = S_MOV_B32 0
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: $exec_lo = S_MOV_B32 -1
+ ; OFF-NEXT: S_ENDPGM 0
+ $exec_lo = S_MOV_B32 0
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ $exec_lo = S_MOV_B32 -1
+ S_ENDPGM 0
+...
+
+---
+name: no_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; ON-LABEL: name: no_arrive
+ ; ON: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: no_arrive
+ ; OFF: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; OFF-NEXT: S_ENDPGM 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ S_ENDPGM 0
+...
+
+---
+name: multiple_arrivals
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; ON-LABEL: name: multiple_arrivals
+ ; ON: bb.0:
+ ; ON-NEXT: successors: %bb.1(0x80000000)
+ ; ON-NEXT: liveins: $vgpr0
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: S_BRANCH %bb.1
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: bb.1:
+ ; ON-NEXT: liveins: $vgpr0
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: multiple_arrivals
+ ; OFF: bb.0:
+ ; OFF-NEXT: successors: %bb.1(0x80000000)
+ ; OFF-NEXT: liveins: $vgpr0
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: S_BRANCH %bb.1
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: bb.1:
+ ; OFF-NEXT: liveins: $vgpr0
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr0
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ liveins: $vgpr0
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; The extra async operation must not relax the user's partial wait.
+ ; ON-LABEL: name: partial_wait
+ ; ON: liveins: $vgpr0, $vgpr1
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: partial_wait
+ ; OFF: liveins: $vgpr0, $vgpr1
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ ; OFF-NEXT: S_ENDPGM 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ S_ENDPGM 0
+...
+
+---
+name: expert_partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; The extra async operation must not relax the user's partial wait.
+ ; ON-LABEL: name: expert_partial_wait
+ ; ON: liveins: $vgpr0, $vgpr1
+ ; ON-NEXT: {{ $}}
+ ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+ ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; ON-NEXT: V_NOP_e32 implicit $exec
+ ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; ON-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+ ; ON-NEXT: $exec_lo = S_MOV_B32 0
+ ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+ ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+ ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ ; ON-NEXT: S_ENDPGM 0
+ ;
+ ; OFF-LABEL: name: expert_partial_wait
+ ; OFF: liveins: $vgpr0, $vgpr1
+ ; OFF-NEXT: {{ $}}
+ ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+ ; OFF-NEXT: V_NOP_e32 implicit $exec
+ ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+ ; OFF-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ ; OFF-NEXT: S_ENDPGM 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+ S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
new file mode 100644
index 00000000000000..0eb8ad252cb4e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
@@ -0,0 +1,24 @@
+; NOTE: Do not autogenerate. Object emission invokes the inline assembler; null emission does not. Check warning presence and absence.
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -filetype=obj -o %t.o < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+
+define amdgpu_kernel void @unprotected_inline_asm() {
+ call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+ ret void
+}
+
+define amdgpu_kernel void @protected_inline_asm() {
+ call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0\0As_mov_b32 s7, exec_lo\0As_mov_b32 exec_lo, 0\0Aglobal_load_async_to_lds_b8 v0, v[0:1], off\0As_mov_b32 exec_lo, s7", "~{v0},~{v1},~{s7},~{memory}"()
+ ret void
+}
+
+; A later inline-asm block cannot protect an arrival in an earlier block.
+define amdgpu_kernel void @split_inline_asm() {
+ call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+ call void asm sideeffect "global_load_async_to_lds_b32 v0, v[0:1], off", "~{v0},~{v1},~{memory}"()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
index c42d8b4c9cc143..168a97ebd53302 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
@@ -85,6 +85,10 @@ define void @taildup_ds_atomic_async_barrier_arrive(ptr addrspace(1) %a, ptr add
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
; GCN-NEXT: ds_atomic_async_barrier_arrive_b64 v2
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 s1, exec_lo
+; GCN-NEXT: s_mov_b32 exec_lo, 0
+; GCN-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT: s_mov_b32 exec_lo, s1
; GCN-NEXT: s_set_pc_i64 s[30:31]
entry:
br i1 %cond, label %bb1, label %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
index 791a2a3a767eb2..59951e39cb06ed 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -12,6 +12,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64(ptr addrspace(3) %bar) {
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
; GCN-NEXT: ds_atomic_async_barrier_arrive_b64 v0
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 s0, exec_lo
+; GCN-NEXT: s_mov_b32 exec_lo, 0
+; GCN-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT: s_mov_b32 exec_lo, s0
; GCN-NEXT: s_set_pc_i64 s[30:31]
entry:
call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
@@ -26,6 +30,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64_off(ptr addrspace(3) %in) {
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
; GCN-NEXT: ds_atomic_async_barrier_arrive_b64 v0 offset:8184
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 s0, exec_lo
+; GCN-NEXT: s_mov_b32 exec_lo, 0
+; GCN-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT: s_mov_b32 exec_lo, s0
; GCN-NEXT: s_set_pc_i64 s[30:31]
entry:
%bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
@@ -44,6 +52,10 @@ define amdgpu_ps void @test_ds_atomic_async_barrier_arrive_b64_s(ptr addrspace(3
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
; GCN-NEXT: ds_atomic_async_barrier_arrive_b64 v0
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 s105, exec_lo
+; GCN-NEXT: s_mov_b32 exec_lo, 0
+; GCN-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT: s_mov_b32 exec_lo, s105
; GCN-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
new file mode 100644
index 00000000000000..8a8bff7573ade5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
@@ -0,0 +1,38 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across the EXEC-clear sequence.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+
+// VGPR bank changes preserve EXEC across the compiler's dummy-load sequence.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// A bank change alone does not establish that EXEC is clear.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// A bank change does not hide a subsequent write to EXEC.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// Other intervening instructions still break the pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
new file mode 100644
index 00000000000000..f1931aebaef82e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
@@ -0,0 +1,27 @@
+// NOTE: Do not autogenerate. Check target-ID XNACK settings and warning locations.
+// RUN: split-file %s %t
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/on.s 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/off.s 2>&1 | count 0
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/on.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/off.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50s-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.5-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+
+// ON: <stdin>:2:1: warning: no recognized replay workaround
+// ANY: <stdin>:1:1: warning: no recognized replay workaround
+
+//--- on.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack+"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- off.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- any.s
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..719e87544c98a9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
@@ -0,0 +1,187 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across sequence boundaries.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | not llvm-mc -triple=amdgpu12.50 -filetype=null --fatal-warnings 2>&1 | FileCheck %s --check-prefix=FATAL --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=true < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null --no-warn < %s 2>&1 | count 0
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | llvm-mc -triple=amdgpu13.10 -mattr=+lds-barrier-arrive-atomic -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -mattr=-async-barrier-arrive-replay-bug -filetype=null < %s 2>&1 | count 0
+// FATAL: <stdin>:1:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+
+// Non-async VMEM ends recognition before the following load.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_nop 0
+global_load_b32 v1, v[2:3], off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// The compiler's dummy-load sequence is recognized without fixing its SGPR.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// Active B8 loads and scalar-address forms are not the recognized dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+global_load_async_to_lds_b8 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+
+// B32 forms are not recognized, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v[2:3], off
+s_endpgm
+
+// Cluster loads do not suppress the warning, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 m0, 0
+s_mov_b32 exec_lo, 0
+cluster_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Clearing another register or using a nonzero EXEC is not the dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 s7, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Intervening instructions break the immediately preceding EXEC-clear pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Source locations do not interrupt a protective sequence.
+.file 1 "async-barrier.s"
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+.loc 1 10 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// One qualifying load can follow several arrivals.
+ds_atomic_async_barrier_arrive_b64 v0
+ds_atomic_async_barrier_arrive_b64 v1
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// B64/B128 forms are not recognized.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b64 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b128 v4, v[2:3], off
+s_endpgm
+
+// Recognize scratch, flat and buffer VMEM as well as global VMEM.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+scratch_load_b32 v1, v2, off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+flat_load_b32 v1, v[2:3]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+buffer_load_b32 v1, v2, s[4:7], s8 offen
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A load beyond a branch or label does not prove protection on every path.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_branch next
+next:
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+another_path:
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Calls, indirect branches, traps and exception returns end recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_swap_pc_i64 s[2:3], s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_pc_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_trap 2
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_rfe_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A conditional branch must not let a fall-through load hide the warning.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_cbranch_scc0 1
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Bytes emitted by a directive interrupt recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+.long 0
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Preserve the original location through macro expansion.
+.macro arrive
+ ds_atomic_async_barrier_arrive_b64 v0
+// WARN: <instantiation>:1:1: warning: no recognized replay workaround
+.endm
+arrive
+s_endpgm
+
+// EOF also diagnoses an outstanding arrival.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
More information about the cfe-commits
mailing list