[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)

Yaxun Liu via cfe-commits cfe-commits at lists.llvm.org
Tue Nov 3 13:47:55 PST 2026


https://github.com/yxsamliu created https://github.com/llvm/llvm-project/pull/238539

Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.

Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.


>From f8361a7e7d813edf75bbaf1677b22848bdd7f62d Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 3 Nov 2026 12:01:44 -0500
Subject: [PATCH] [AMDGPU] Add an async barrier arrival workaround

Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.

Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.
---
 .../AMDGPU/async-barrier-replay-warning.c     |  20 +
 .../amdgpu-async-barrier-replay-warning.s     |  15 +
 llvm/docs/AMDGPUUsage.md                      |  28 +
 llvm/docs/ReleaseNotes.md                     |   6 +
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  81 +++
 llvm/lib/Target/AMDGPU/GCNFeatures.td         |   7 +
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |   6 +
 llvm/lib/Target/AMDGPU/SIFrameLowering.cpp    |  36 +-
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  38 ++
 llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp  |  11 +-
 .../async-barrier-arrive-replay-vgpr-msb.mir  |  32 ++
 .../async-barrier-arrive-replay-whole-wave.ll | 494 ++++++++++++++++++
 .../AMDGPU/async-barrier-arrive-replay.ll     |  46 ++
 .../AMDGPU/async-barrier-arrive-replay.mir    | 269 ++++++++++
 .../AMDGPU/async-barrier-replay-inline-asm.ll |  24 +
 .../AMDGPU/ds-atomic-barrier-convergent.ll    |   4 +
 ...dgcn.ds.atomic.async.barrier.arrive.b64.ll |  12 +
 .../async-barrier-replay-warning-vgpr-msb.s   |  38 ++
 .../async-barrier-replay-warning-xnack.s      |  27 +
 .../MC/AMDGPU/async-barrier-replay-warning.s  | 187 +++++++
 20 files changed, 1370 insertions(+), 11 deletions(-)
 create mode 100644 clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
 create mode 100644 clang/test/Driver/amdgpu-async-barrier-replay-warning.s
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning.s

diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
new file mode 100644
index 00000000000000..b0b57f42c26bd0
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
@@ -0,0 +1,20 @@
+// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
+
+void unprotected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
+// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+}
+
+void protected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
+                   "s_mov_b32 s7, exec_lo\n"
+                   "s_mov_b32 exec_lo, 0\n"
+                   "global_load_async_to_lds_b8 v0, v[0:1], off\n"
+                   "s_mov_b32 exec_lo, s7"
+                   ::: "v0", "v1", "s7", "memory");
+}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..3a1549c3b325e3
--- /dev/null
+++ b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
@@ -0,0 +1,15 @@
+// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
+// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
+// RUN: cat %s >> %t.s
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index fe5832b286a2b3..86ed4db05ce05f 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17109,6 +17109,34 @@ It supports AMDGCN GFX6-GFX11.
 
 This section describes general syntax for instructions and operands.
 
+(amdgpu-async-barrier-replay-warning)=
+
+#### Async Barrier Arrival Warning
+
+The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
+unless XNACK is explicitly disabled. The assembler checks hand-written and
+inline assembly for the same sequence. This check is also omitted when XNACK
+is explicitly disabled.
+
+The assembler warns by default when it cannot recognize the compiler's dummy
+form: `s_mov_b32 exec_lo, 0` followed by
+`global_load_async_to_lds_b8` with a vector address, issued after an arrival
+and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
+not change EXEC, may intervene between the EXEC clear and the load.
+Other load opcodes, scalar-address forms,
+and active loads do not suppress the warning. Recognition stops at labels,
+control-flow instructions, and the end of assembly. Assembler directives
+can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
+block is checked separately. A warning reports missing recognized protection;
+absence of a warning does not prove control-flow safety, correct EXEC
+preservation, or correct async wait counts.
+
+Disable only this check in Clang with
+`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
+`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
+suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
+errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
+
 #### Instructions
 
 An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 75318319d88b92..bbf4618825417f 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,12 @@ Makes programs 10x faster by doing Special New Thing.
 
 ### Changes to the AMDGPU Backend
 
+* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
+  assembler warnings for unrecognized sequences in handwritten or inline
+  assembly. Both are omitted when XNACK is explicitly disabled. See
+  {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
+  warning controls.
+
 * Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
   and `amdgpu.sramecc` module flags.
 * `llvm.amdgcn.make.buffer.rsrc` now accepts any integer width for its
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index a8c6d5ef457639..19314b0f240be3 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
 #include "Utils/AMDKernelCodeTUtils.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
@@ -40,10 +41,12 @@
 #include "llvm/Support/AMDGPUObjLinkingInfo.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 #include <optional>
+#include <utility>
 
 using namespace llvm;
 using namespace llvm::AMDGPU;
@@ -51,6 +54,12 @@ using namespace llvm::amdhsa;
 
 namespace {
 
+static cl::opt<bool> WarnAsyncBarrierReplay(
+    "amdgpu-asm-warn-async-barrier-replay",
+    cl::desc("Warn about async barrier arrivals without a recognized replay "
+             "workaround in assembly"),
+    cl::init(true), cl::Hidden);
+
 class AMDGPUAsmParser;
 
 enum RegisterKind {
@@ -1135,6 +1144,12 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   const AMDGPU::GPUKind Gfx;
   const AMDGPU::IsaVersion ISA;
 
+  SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
+  bool AsyncBarrierExecCleared = false;
+
+  void checkAsyncBarrierReplay(const MCInst &Inst);
+  void warnPendingAsyncBarrierArrivals();
+
   /// @name Auto-generated Match Functions
   /// {
 
@@ -5839,6 +5854,64 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
   return false;
 }
 
+void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
+  for (SMLoc Loc : PendingAsyncBarrierArrivals) {
+    Warning(Loc, "no recognized replay workaround after "
+                 "'ds_atomic_async_barrier_arrive_b64' on this GPU");
+  }
+  PendingAsyncBarrierArrivals.clear();
+  AsyncBarrierExecCleared = false;
+}
+
+void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
+  if (!WarnAsyncBarrierReplay ||
+      !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
+      getTargetStreamer().getTargetID()->getXnackSetting() ==
+          AMDGPU::TargetIDSetting::Off)
+    return;
+
+  const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
+  switch (Inst.getOpcode()) {
+  case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
+    PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
+    return;
+  // These instructions change control flow but their MC descriptors do not
+  // carry branch, return, or trap flags.
+  case AMDGPU::S_RFE_B64_gfx1250:
+  case AMDGPU::S_TRAP_gfx12:
+    warnPendingAsyncBarrierArrivals();
+    return;
+  case AMDGPU::S_MOV_B32_gfx12:
+    if (!PendingAsyncBarrierArrivals.empty()) {
+      const MCOperand &Dst = Inst.getOperand(0);
+      const MCOperand &Src = Inst.getOperand(1);
+      AsyncBarrierExecCleared = Dst.isReg() &&
+                                mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
+                                Src.isImm() && Src.getImm() == 0;
+    }
+    return;
+  case AMDGPU::S_SET_VGPR_MSB_gfx12:
+    // VGPR encoding can insert this between clearing EXEC and the dummy load.
+    AsyncBarrierExecCleared = ExecCleared;
+    return;
+  case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
+    // Recognize only the compiler's EXEC-zero B8 dummy form.
+    if (ExecCleared)
+      PendingAsyncBarrierArrivals.clear();
+    return;
+  }
+
+  if (PendingAsyncBarrierArrivals.empty())
+    return;
+
+  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
+  // Check only a straight-line sequence. Labels and control-flow changes end
+  // recognition rather than allowing a load on another path to hide a warning.
+  if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
+      (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
+    warnPendingAsyncBarrierArrivals();
+}
+
 bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                               OperandVector &Operands,
                                               MCStreamer &Out,
@@ -5878,6 +5951,7 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       return true;
     }
     emitTargetDirective();
+    checkAsyncBarrierReplay(Inst);
     Out.emitInstruction(Inst, getSTI());
     // Record for kernel prologue checking.
     OpcodeStream.push_back(Inst.getOpcode());
@@ -7027,6 +7101,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
 }
 
 void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
+  warnPendingAsyncBarrierArrivals();
   // Record every parsed label in the timeline so that, at end of file, the
   // instructions following a kernel's label can be located regardless of
   // whether the .amdhsa_kernel directive came before or after the label.
@@ -7057,6 +7132,7 @@ void AMDGPUAsmParser::checkKernelPrologues() {
 }
 
 void AMDGPUAsmParser::onEndOfFile() {
+  warnPendingAsyncBarrierArrivals();
   emitTargetDirective();
   checkKernelPrologues();
   if (InfoData)
@@ -7066,6 +7142,11 @@ void AMDGPUAsmParser::onEndOfFile() {
 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getString();
 
+  // Debug locations do not interrupt the instruction sequence. Other
+  // directives may change sections or emit bytes outside this parser.
+  if (IDVal != ".loc" && IDVal != ".file")
+    warnPendingAsyncBarrierArrivals();
+
   if (isHsaAbi(getSTI())) {
     if (IDVal == ".amdhsa_kernel")
       return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index 313f67ea31e22f..cc8ed4cdd251f3 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -265,6 +265,12 @@ def FeaturePreciseMemory
     : SubtargetFeature<"precise-memory", "EnablePreciseMemory",
                        "true", "Enable precise memory mode">;
 
+defm AsyncBarrierArriveReplayBug : AMDGPUSubtargetFeature<
+  "async-barrier-arrive-replay-bug",
+  "Enable the async barrier arrival workaround",
+  /*GenPredicate=*/0, /*GenAssemblerPredicate=*/0, [], InlineIgnore
+>;
+
 defm SGPRInitBug : AMDGPUSubtargetFeature<"sgpr-init-bug",
   "VI SGPR initialization bug requiring a fixed SGPR allocation size",
   /*GenPredicate=*/1, /*GenAssemblerPredicate=*/1, [], InlineIgnore
@@ -2493,6 +2499,7 @@ def FeatureISAVersion12 : FeatureSet<
 
 def FeatureISAVersion12_50_Common : FeatureSet<
   [FeatureGFX12,
+   FeatureAsyncBarrierArriveReplayBug,
    FeatureGFX1250Insts,
    FeatureBackOffBarrier,
    FeatureRequiresAlignedVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 38604fb4741b36..07829fcc1486cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -345,6 +345,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool isXNACKEnabled() const { return TargetID.isXnackOnOrAny(); }
 
+  bool needsAsyncBarrierArriveReplayWorkaround() const {
+    // Generic 12.5 code has no explicit XNACK setting; keep the workaround.
+    return hasAsyncBarrierArriveReplayBug() &&
+           TargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Off;
+  }
+
   bool hasRelaxedBufferOOBMode() const { return BufferOOBRelaxed; }
   bool hasRelaxedTBufferOOBMode() const { return TBufferOOBRelaxed; }
 
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index cb51eeb8b13c96..ce761f94b97a2f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1859,7 +1859,20 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
       // If found any unused scratch SGPR, reserve the register itself for Exec
       // copy and there is no need for any spills in that case.
       MFI->setSGPRForEXECCopy(UnusedScratchReg);
-      MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
+      // Only rewrite EXEC saves and restores. Other uses, such as inline
+      // assembly operands and their callee saves, name fixed registers.
+      const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+      for (MachineOperand &Op :
+           make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
+        MachineInstr &MI = *Op.getParent();
+        if (MI.getOpcode() == LMC.OrSaveExecOpc ||
+            (MI.getOpcode() == LMC.MovOpc &&
+             (MI.getOperand(0).getReg() == LMC.ExecReg ||
+              (MI.getOperand(1).isReg() &&
+               MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+          Op.setReg(UnusedScratchReg);
+        }
+      }
       LiveUnits.addReg(UnusedScratchReg);
     } else {
       // Needs spill.
@@ -1933,19 +1946,23 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
   const SIInstrInfo *TII = ST.getInstrInfo();
   bool NeedExecCopyReservedReg = false;
+  bool NeedExecCopyForAsyncBarrier = false;
 
   MachineInstr *ReturnMI = nullptr;
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
       // TODO: Walking through all MBBs here would be a bad heuristic. Better
       // handle them elsewhere.
-      if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+      if (TII->isWWMRegSpillOpcode(MI.getOpcode())) {
         NeedExecCopyReservedReg = true;
-      else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
-               MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
-               MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
-               (MFI->isChainFunction() &&
-                TII->isChainCallOpcode(MI.getOpcode()))) {
+      } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
+                 ST.needsAsyncBarrierArriveReplayWorkaround()) {
+        NeedExecCopyForAsyncBarrier = true;
+      } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
+                 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
+                 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
+                 (MFI->isChainFunction() &&
+                  TII->isChainCallOpcode(MI.getOpcode()))) {
         // We expect all return to be the same size.
         assert(!ReturnMI ||
                (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
@@ -2013,7 +2030,10 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   if (!ST.hasGFX90AInsts())
     SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
 
-  determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
+  // Whole-wave spills use the original EXEC mask from the first argument, but
+  // the barrier workaround needs separate storage for the current EXEC mask.
+  determinePrologEpilogSGPRSaves(
+      MF, SavedVGPRs, NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
 
   // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
   // allow the default insertion to handle them.
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bd2815d0491b5b..9fa338a3f1826f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 
 using namespace llvm;
@@ -3497,6 +3498,43 @@ bool SIInsertWaitcnts::run() {
 
   bool Modified = false;
 
+  if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
+    // Collect arrivals before inserting new definitions of ASYNCcnt.
+    SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+    for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+      if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+        AsyncBarrierArrivals.push_back(&MI);
+    }
+
+    if (!AsyncBarrierArrivals.empty()) {
+      const Register SavedExec = MFI->getSGPRForEXECCopy();
+      if (!SavedExec) {
+        report_fatal_error(
+            "missing EXEC copy SGPR for async barrier workaround");
+      }
+
+      for (MachineInstr *MI : AsyncBarrierArrivals) {
+        MachineBasicBlock &MBB = *MI->getParent();
+        const DebugLoc &DL = MI->getDebugLoc();
+        // Insert after scheduling so the dummy stays after its arrival.
+        auto InsertPt = std::next(MI->getIterator());
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+            .addReg(AMDGPU::EXEC_LO);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addImm(0);
+        // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+            .addReg(AMDGPU::VGPR0, RegState::Undef)
+            .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+            .addImm(0)
+            .addImm(0);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addReg(SavedExec);
+        Modified = true;
+      }
+    }
+  }
+
   MachineBasicBlock &EntryBB = MF.front();
 
   if (!MFI->isEntryFunction() &&
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index d0a57e163d29a0..f48e2194242a42 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -619,9 +619,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
     if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
                              TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
       FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
-  } else {
-    // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
-    // spills/copies. Reset the SGPR reserved for EXEC copy.
+  } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
+             none_of(MRI.def_instructions(AMDGPU::ASYNCcnt),
+                     [](const MachineInstr &MI) {
+                       return MI.getOpcode() ==
+                              AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
+                     })) {
+    // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
+    // needs the reserved register for EXEC copy.
     FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
new file mode 100644
index 00000000000000..155f0d3b296a3a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
@@ -0,0 +1,32 @@
+# NOTE: Do not autogenerate. Check the bank switch inside the workaround and assemble compiler output with fatal warnings.
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
+
+--- |
+  define amdgpu_cs void @high_vgpr_arrive() { ret void }
+...
+
+---
+name: high_vgpr_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr256
+    ; The arrival needs a high VGPR, but the dummy uses v[0:1]. The late
+    ; encoding pass resets the bank after EXEC is cleared.
+    ; CHECK-LABEL: high_vgpr_arrive:
+    ; CHECK: s_set_vgpr_msb 1
+    ; CHECK-NEXT: ds_atomic_async_barrier_arrive_b64 v0 /*v256*/
+    ; CHECK: s_mov_b32 s105, exec_lo
+    ; CHECK-NEXT: s_mov_b32 exec_lo, 0
+    ; CHECK-NEXT: s_set_vgpr_msb 0x100
+    ; CHECK-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+    ; CHECK-NEXT: s_mov_b32 exec_lo, s105
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr256, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
new file mode 100644
index 00000000000000..5998e619fbcab7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
@@ -0,0 +1,494 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+
+; The caller initializes the LDS barrier at offset zero. Only the active-mask
+; argument is used, since ordinary arguments are poison in inactive lanes.
+; gfx1250 and gfx1251 do not support wave64.
+
+; The barrier must use a separate scratch SGPR from the saved incoming EXEC,
+; even when the active argument has no explicit uses.
+define amdgpu_gfx_whole_wave void @whole_wave_barrier(i1 %active) {
+; ON-LABEL: whole_wave_barrier:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mov_b32_e32 v0, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_barrier:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mov_b32_e32 v0, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret void
+}
+
+; Moving the EXEC scratch must preserve fixed-register assembly operands and the
+; corresponding callee saves, in both whole-wave and ordinary functions.
+define amdgpu_gfx_whole_wave i32 @whole_wave_clobber_exec_scratch(i1 %active) {
+; ON-LABEL: whole_wave_clobber_exec_scratch:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT:    scratch_store_b32 off, v2, s32 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v1, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_writelane_b32 v2, s105, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    s_mov_b32 s105, 0
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT:    v_readlane_b32 s105, v2, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT:    scratch_load_b32 v2, off, s32 nv
+; ON-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; ON-NEXT:    scratch_load_b32 v1, off, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_clobber_exec_scratch:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v1, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_writelane_b32 v2, s105, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    s_mov_b32 s105, 0
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT:    v_readlane_b32 s105, v2, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 nv
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; OFF-NEXT:    scratch_load_b32 v1, off, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret i32 %value
+}
+
+define i32 @callable_clobber_exec_scratch() {
+; ON-LABEL: callable_clobber_exec_scratch:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    v_writelane_b32 v2, s105, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    s_mov_b32 s105, 0
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT:    v_readlane_b32 s105, v2, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s0, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: callable_clobber_exec_scratch:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    v_writelane_b32 v2, s105, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    s_mov_b32 s105, 0
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT:    v_readlane_b32 s105, v2, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret i32 %value
+}
+
+; Keep the active argument live across the barrier and exercise SGPR and VGPR
+; callee saves as well as the reserved scratch register.
+define amdgpu_gfx_whole_wave i32 @whole_wave_active(i1 %active) {
+; SDAG-LABEL: whole_wave_active:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT:    s_wait_kmcnt 0x0
+; SDAG-NEXT:    s_xor_saveexec_b32 s0, -1
+; SDAG-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; SDAG-NEXT:    scratch_store_b32 off, v1, s32 nv
+; SDAG-NEXT:    ; meta instruction
+; SDAG-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_mov_b32 exec_lo, -1
+; SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAG-NEXT:    v_writelane_b32 v1, s68, 0
+; SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    ;;#ASMSTART
+; SDAG-NEXT:    ; clobber CSR
+; SDAG-NEXT:    ;;#ASMEND
+; SDAG-NEXT:    s_and_b32 s1, s0, exec_lo
+; SDAG-NEXT:    s_cselect_b32 s1, 42, 7
+; SDAG-NEXT:    v_readlane_b32 s68, v1, 0
+; SDAG-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; SDAG-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT:    s_mov_b32 s2, exec_lo
+; SDAG-NEXT:    s_mov_b32 exec_lo, 0
+; SDAG-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; SDAG-NEXT:    s_mov_b32 exec_lo, s2
+; SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAG-NEXT:    v_mov_b32_e32 v0, s1
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_xor_b32 exec_lo, s0, -1
+; SDAG-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; SDAG-NEXT:    scratch_load_b32 v1, off, s32 nv
+; SDAG-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_mov_b32 exec_lo, s0
+; SDAG-NEXT:    s_wait_loadcnt 0x0
+; SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; SDAGOFF-LABEL: whole_wave_active:
+; SDAGOFF:       ; %bb.0:
+; SDAGOFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAGOFF-NEXT:    s_wait_kmcnt 0x0
+; SDAGOFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; SDAGOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; SDAGOFF-NEXT:    scratch_store_b32 off, v1, s32 nv
+; SDAGOFF-NEXT:    ; meta instruction
+; SDAGOFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAGOFF-NEXT:    s_mov_b32 exec_lo, -1
+; SDAGOFF-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAGOFF-NEXT:    v_writelane_b32 v1, s68, 0
+; SDAGOFF-NEXT:    v_mov_b32_e32 v0, 0
+; SDAGOFF-NEXT:    ;;#ASMSTART
+; SDAGOFF-NEXT:    ; clobber CSR
+; SDAGOFF-NEXT:    ;;#ASMEND
+; SDAGOFF-NEXT:    s_and_b32 s1, s0, exec_lo
+; SDAGOFF-NEXT:    s_cselect_b32 s1, 42, 7
+; SDAGOFF-NEXT:    v_readlane_b32 s68, v1, 0
+; SDAGOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; SDAGOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAGOFF-NEXT:    v_mov_b32_e32 v0, s1
+; SDAGOFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; SDAGOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; SDAGOFF-NEXT:    scratch_load_b32 v1, off, s32 nv
+; SDAGOFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAGOFF-NEXT:    s_mov_b32 exec_lo, s0
+; SDAGOFF-NEXT:    s_wait_loadcnt 0x0
+; SDAGOFF-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: whole_wave_active:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_kmcnt 0x0
+; GISEL-NEXT:    s_xor_saveexec_b32 s0, -1
+; GISEL-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; GISEL-NEXT:    scratch_store_b32 off, v1, s32 nv
+; GISEL-NEXT:    ; meta instruction
+; GISEL-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_mov_b32 exec_lo, -1
+; GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISEL-NEXT:    v_writelane_b32 v1, s68, 0
+; GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    ;;#ASMSTART
+; GISEL-NEXT:    ; clobber CSR
+; GISEL-NEXT:    ;;#ASMEND
+; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISEL-NEXT:    v_readlane_b32 s68, v1, 0
+; GISEL-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; GISEL-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT:    s_mov_b32 s1, exec_lo
+; GISEL-NEXT:    s_mov_b32 exec_lo, 0
+; GISEL-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_xor_b32 exec_lo, s0, -1
+; GISEL-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; GISEL-NEXT:    scratch_load_b32 v1, off, s32 nv
+; GISEL-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_mov_b32 exec_lo, s0
+; GISEL-NEXT:    s_wait_loadcnt 0x0
+; GISEL-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GISELOFF-LABEL: whole_wave_active:
+; GISELOFF:       ; %bb.0:
+; GISELOFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISELOFF-NEXT:    s_wait_kmcnt 0x0
+; GISELOFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; GISELOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; GISELOFF-NEXT:    scratch_store_b32 off, v1, s32 nv
+; GISELOFF-NEXT:    ; meta instruction
+; GISELOFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; GISELOFF-NEXT:    s_mov_b32 exec_lo, -1
+; GISELOFF-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISELOFF-NEXT:    v_writelane_b32 v1, s68, 0
+; GISELOFF-NEXT:    v_mov_b32_e32 v0, 0
+; GISELOFF-NEXT:    ;;#ASMSTART
+; GISELOFF-NEXT:    ; clobber CSR
+; GISELOFF-NEXT:    ;;#ASMEND
+; GISELOFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISELOFF-NEXT:    v_readlane_b32 s68, v1, 0
+; GISELOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; GISELOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISELOFF-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; GISELOFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; GISELOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; GISELOFF-NEXT:    scratch_load_b32 v1, off, s32 nv
+; GISELOFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; GISELOFF-NEXT:    s_mov_b32 exec_lo, s0
+; GISELOFF-NEXT:    s_wait_loadcnt 0x0
+; GISELOFF-NEXT:    s_set_pc_i64 s[30:31]
+  call void asm sideeffect "; clobber CSR", "~{v40},~{s68}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  %result = select i1 %active, i32 42, i32 7
+  ret i32 %result
+}
+
+; A divergent region has a current EXEC mask distinct from the original
+; active argument. Preserve both through the workaround.
+define amdgpu_gfx_whole_wave i32 @whole_wave_divergent(i1 %active) {
+; ON-LABEL: whole_wave_divergent:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mbcnt_lo_u32_b32 v0, -1, 0
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; ON-NEXT:    v_and_b32_e32 v0, 1, v0
+; ON-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; ON-NEXT:    s_cbranch_execz .LBB4_2
+; ON-NEXT:  ; %bb.1: ; %arrive
+; ON-NEXT:    v_mov_b32_e32 v0, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s2, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s2
+; ON-NEXT:  .LBB4_2: ; %exit
+; ON-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; ON-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; ON-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_divergent:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mbcnt_lo_u32_b32 v0, -1, 0
+; OFF-NEXT:    s_mov_b32 s1, exec_lo
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; OFF-NEXT:    v_and_b32_e32 v0, 1, v0
+; OFF-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; OFF-NEXT:  ; %bb.1: ; %arrive
+; OFF-NEXT:    v_mov_b32_e32 v0, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:  ; %bb.2: ; %exit
+; OFF-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; OFF-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %parity = and i32 %lane, 1
+  %cond = icmp eq i32 %parity, 0
+  br i1 %cond, label %arrive, label %exit
+
+arrive:
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  br label %exit
+
+exit:
+  %result = select i1 %active, i32 42, i32 7
+  ret i32 %result
+}
+
+; Keep both dummy-load address registers live across the workaround.
+define amdgpu_gfx_whole_wave void @whole_wave_live_vgprs(i1 %active) {
+; ON-LABEL: whole_wave_live_vgprs:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v1, s32 offset:4 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v2, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mov_b32_e32 v2, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    v_mov_b32_e32 v0, 0x7b
+; ON-NEXT:    v_mov_b32_e32 v1, 0x1c8
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    ; use v[0:1]
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv
+; ON-NEXT:    scratch_load_b32 v1, off, s32 offset:4 nv
+; ON-NEXT:    scratch_load_b32 v2, off, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_live_vgprs:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v1, s32 offset:4 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mov_b32_e32 v2, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    v_mov_b32_e32 v0, 0x7b
+; OFF-NEXT:    v_mov_b32_e32 v1, 0x1c8
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    ; use v[0:1]
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv
+; OFF-NEXT:    scratch_load_b32 v1, off, s32 offset:4 nv
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %values = call <2 x i32> asm sideeffect "v_mov_b32 v0, 123\0Av_mov_b32 v1, 456", "={v[0:1]}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  call void asm sideeffect "; use $0", "{v[0:1]}"(<2 x i32> %values)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
new file mode 100644
index 00000000000000..6158c7ec97427e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
@@ -0,0 +1,46 @@
+; NOTE: Do not autogenerate. Check workaround ordering on affected targets and absence on unaffected targets.
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu13.10-amd-amdhsa -mattr=+lds-barrier-arrive-atomic < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.5-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-async-barrier-arrive-replay-bug < %s | FileCheck --check-prefix=UNAFFECTED %s
+
+; Check an LDS barrier arrival followed by a global load.
+ at bar = internal addrspace(3) global i64 undef, align 8
+
+; AFFECTED-LABEL: replay_window:
+; AFFECTED: ds_atomic_async_barrier_arrive_b64
+; AFFECTED-NOT: global_load_b32
+; AFFECTED: s_mov_b32 [[SAVED:s[0-9]+]], exec_lo
+; AFFECTED-NEXT: s_mov_b32 exec_lo, 0
+; AFFECTED-NEXT: global_load_async_to_lds_b8
+; AFFECTED-NEXT: s_mov_b32 exec_lo, [[SAVED]]
+; AFFECTED: global_load_b32
+;
+; UNAFFECTED-LABEL: replay_window:
+; UNAFFECTED: ds_atomic_async_barrier_arrive_b64
+; UNAFFECTED-NOT: global_load_async_to_lds
+; UNAFFECTED: global_load_b32
+define amdgpu_kernel void @replay_window(ptr addrspace(1) %src,
+                                         ptr addrspace(1) %dst) {
+entry:
+  store i64 0, ptr addrspace(3) @bar, align 8
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) @bar)
+  %value = load volatile i32, ptr addrspace(1) %src, align 4
+  store volatile i32 %value, ptr addrspace(1) %dst, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
new file mode 100644
index 00000000000000..09eb65e1ac5cd8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
@@ -0,0 +1,269 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+
+--- |
+  define amdgpu_kernel void @live_scc() { ret void }
+  define amdgpu_kernel void @empty_exec() { ret void }
+  define amdgpu_kernel void @no_arrive() { ret void }
+  define amdgpu_kernel void @multiple_arrivals() { ret void }
+  define amdgpu_kernel void @partial_wait() { ret void }
+  define amdgpu_kernel void @expert_partial_wait() #0 { ret void }
+  attributes #0 = { "amdgpu-expert-scheduling-mode"="true" }
+...
+
+---
+name: live_scc
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; ON-LABEL: name: live_scc
+    ; ON: liveins: $sgpr0, $vgpr0
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: live_scc
+    ; OFF: liveins: $sgpr0, $vgpr0
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    ; OFF-NEXT: S_ENDPGM 0
+    S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    S_ENDPGM 0
+...
+
+---
+name: empty_exec
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; ON-LABEL: name: empty_exec
+    ; ON: liveins: $vgpr0
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: $exec_lo = S_MOV_B32 -1
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: empty_exec
+    ; OFF: liveins: $vgpr0
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: $exec_lo = S_MOV_B32 0
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: $exec_lo = S_MOV_B32 -1
+    ; OFF-NEXT: S_ENDPGM 0
+    $exec_lo = S_MOV_B32 0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $exec_lo = S_MOV_B32 -1
+    S_ENDPGM 0
+...
+
+---
+name: no_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    ; ON-LABEL: name: no_arrive
+    ; ON: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: no_arrive
+    ; OFF: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    S_ENDPGM 0
+...
+
+---
+name: multiple_arrivals
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  ; ON-LABEL: name: multiple_arrivals
+  ; ON: bb.0:
+  ; ON-NEXT:   successors: %bb.1(0x80000000)
+  ; ON-NEXT:   liveins: $vgpr0
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT:   S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+  ; ON-NEXT:   $sgpr64_sgpr65 = S_MOV_B64 0
+  ; ON-NEXT:   V_NOP_e32 implicit $exec
+  ; ON-NEXT:   GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+  ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   S_BRANCH %bb.1
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT: bb.1:
+  ; ON-NEXT:   liveins: $vgpr0
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   S_ENDPGM 0
+  ;
+  ; OFF-LABEL: name: multiple_arrivals
+  ; OFF: bb.0:
+  ; OFF-NEXT:   successors: %bb.1(0x80000000)
+  ; OFF-NEXT:   liveins: $vgpr0
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT:   $sgpr64_sgpr65 = S_MOV_B64 0
+  ; OFF-NEXT:   V_NOP_e32 implicit $exec
+  ; OFF-NEXT:   GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+  ; OFF-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; OFF-NEXT:   S_BRANCH %bb.1
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT: bb.1:
+  ; OFF-NEXT:   liveins: $vgpr0
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; OFF-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    liveins: $vgpr0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; The extra async operation must not relax the user's partial wait.
+    ; ON-LABEL: name: partial_wait
+    ; ON: liveins: $vgpr0, $vgpr1
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: partial_wait
+    ; OFF: liveins: $vgpr0, $vgpr1
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    S_ENDPGM 0
+...
+
+---
+name: expert_partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; The extra async operation must not relax the user's partial wait.
+    ; ON-LABEL: name: expert_partial_wait
+    ; ON: liveins: $vgpr0, $vgpr1
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: expert_partial_wait
+    ; OFF: liveins: $vgpr0, $vgpr1
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
new file mode 100644
index 00000000000000..0eb8ad252cb4e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
@@ -0,0 +1,24 @@
+; NOTE: Do not autogenerate. Object emission invokes the inline assembler; null emission does not. Check warning presence and absence.
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -filetype=obj -o %t.o < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+
+define amdgpu_kernel void @unprotected_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+  ret void
+}
+
+define amdgpu_kernel void @protected_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0\0As_mov_b32 s7, exec_lo\0As_mov_b32 exec_lo, 0\0Aglobal_load_async_to_lds_b8 v0, v[0:1], off\0As_mov_b32 exec_lo, s7", "~{v0},~{v1},~{s7},~{memory}"()
+  ret void
+}
+
+; A later inline-asm block cannot protect an arrival in an earlier block.
+define amdgpu_kernel void @split_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+  call void asm sideeffect "global_load_async_to_lds_b32 v0, v[0:1], off", "~{v0},~{v1},~{memory}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
index c42d8b4c9cc143..168a97ebd53302 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
@@ -85,6 +85,10 @@ define void @taildup_ds_atomic_async_barrier_arrive(ptr addrspace(1) %a, ptr add
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s1, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s1
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   br i1 %cond, label %bb1, label %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
index 791a2a3a767eb2..59951e39cb06ed 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -12,6 +12,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64(ptr addrspace(3) %bar) {
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s0, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s0
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
@@ -26,6 +30,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64_off(ptr addrspace(3) %in) {
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0 offset:8184
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s0, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s0
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
@@ -44,6 +52,10 @@ define amdgpu_ps void @test_ds_atomic_async_barrier_arrive_b64_s(ptr addrspace(3
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s105, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s105
 ; GCN-NEXT:    s_endpgm
 entry:
   call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
new file mode 100644
index 00000000000000..8a8bff7573ade5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
@@ -0,0 +1,38 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across the EXEC-clear sequence.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+
+// VGPR bank changes preserve EXEC across the compiler's dummy-load sequence.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// A bank change alone does not establish that EXEC is clear.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// A bank change does not hide a subsequent write to EXEC.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// Other intervening instructions still break the pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
new file mode 100644
index 00000000000000..f1931aebaef82e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
@@ -0,0 +1,27 @@
+// NOTE: Do not autogenerate. Check target-ID XNACK settings and warning locations.
+// RUN: split-file %s %t
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/on.s 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/off.s 2>&1 | count 0
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/on.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/off.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50s-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.5-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+
+// ON: <stdin>:2:1: warning: no recognized replay workaround
+// ANY: <stdin>:1:1: warning: no recognized replay workaround
+
+//--- on.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack+"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- off.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- any.s
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..719e87544c98a9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
@@ -0,0 +1,187 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across sequence boundaries.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | not llvm-mc -triple=amdgpu12.50 -filetype=null --fatal-warnings 2>&1 | FileCheck %s --check-prefix=FATAL --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=true < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null --no-warn < %s 2>&1 | count 0
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | llvm-mc -triple=amdgpu13.10 -mattr=+lds-barrier-arrive-atomic -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -mattr=-async-barrier-arrive-replay-bug -filetype=null < %s 2>&1 | count 0
+// FATAL: <stdin>:1:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+
+// Non-async VMEM ends recognition before the following load.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_nop 0
+global_load_b32 v1, v[2:3], off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// The compiler's dummy-load sequence is recognized without fixing its SGPR.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// Active B8 loads and scalar-address forms are not the recognized dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+global_load_async_to_lds_b8 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+
+// B32 forms are not recognized, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v[2:3], off
+s_endpgm
+
+// Cluster loads do not suppress the warning, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 m0, 0
+s_mov_b32 exec_lo, 0
+cluster_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Clearing another register or using a nonzero EXEC is not the dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 s7, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Intervening instructions break the immediately preceding EXEC-clear pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Source locations do not interrupt a protective sequence.
+.file 1 "async-barrier.s"
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+.loc 1 10 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// One qualifying load can follow several arrivals.
+ds_atomic_async_barrier_arrive_b64 v0
+ds_atomic_async_barrier_arrive_b64 v1
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// B64/B128 forms are not recognized.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b64 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b128 v4, v[2:3], off
+s_endpgm
+
+// Recognize scratch, flat and buffer VMEM as well as global VMEM.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+scratch_load_b32 v1, v2, off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+flat_load_b32 v1, v[2:3]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+buffer_load_b32 v1, v2, s[4:7], s8 offen
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A load beyond a branch or label does not prove protection on every path.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_branch next
+next:
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+another_path:
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Calls, indirect branches, traps and exception returns end recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_swap_pc_i64 s[2:3], s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_pc_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_trap 2
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_rfe_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A conditional branch must not let a fall-through load hide the warning.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_cbranch_scc0 1
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Bytes emitted by a directive interrupt recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+.long 0
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Preserve the original location through macro expansion.
+.macro arrive
+  ds_atomic_async_barrier_arrive_b64 v0
+// WARN: <instantiation>:1:1: warning: no recognized replay workaround
+.endm
+arrive
+s_endpgm
+
+// EOF also diagnoses an outstanding arrival.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround



More information about the cfe-commits mailing list