[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)

Yaxun Liu via cfe-commits cfe-commits at lists.llvm.org
Sat Nov 7 05:46:46 PST 2026


https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/238539

>From e5fcee36e9910432dbcff0c8f7355edac499def7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 3 Nov 2026 12:01:44 -0500
Subject: [PATCH 1/4] [AMDGPU] Add an async barrier arrival workaround

Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.

Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.
---
 .../AMDGPU/async-barrier-replay-warning.c     |  20 +
 .../amdgpu-async-barrier-replay-warning.s     |  15 +
 llvm/docs/AMDGPUUsage.md                      |  28 +
 llvm/docs/ReleaseNotes.md                     |   6 +
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  81 +++
 llvm/lib/Target/AMDGPU/GCNFeatures.td         |   7 +
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |   6 +
 llvm/lib/Target/AMDGPU/SIFrameLowering.cpp    |  36 +-
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  38 ++
 llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp  |  11 +-
 .../async-barrier-arrive-replay-vgpr-msb.mir  |  32 ++
 .../async-barrier-arrive-replay-whole-wave.ll | 494 ++++++++++++++++++
 .../AMDGPU/async-barrier-arrive-replay.ll     |  46 ++
 .../AMDGPU/async-barrier-arrive-replay.mir    | 269 ++++++++++
 .../AMDGPU/async-barrier-replay-inline-asm.ll |  24 +
 .../AMDGPU/ds-atomic-barrier-convergent.ll    |   4 +
 ...dgcn.ds.atomic.async.barrier.arrive.b64.ll |  12 +
 .../async-barrier-replay-warning-vgpr-msb.s   |  38 ++
 .../async-barrier-replay-warning-xnack.s      |  27 +
 .../MC/AMDGPU/async-barrier-replay-warning.s  | 187 +++++++
 20 files changed, 1370 insertions(+), 11 deletions(-)
 create mode 100644 clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
 create mode 100644 clang/test/Driver/amdgpu-async-barrier-replay-warning.s
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
 create mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning.s

diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
new file mode 100644
index 00000000000000..b0b57f42c26bd0
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
@@ -0,0 +1,20 @@
+// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
+
+void unprotected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
+// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+}
+
+void protected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
+                   "s_mov_b32 s7, exec_lo\n"
+                   "s_mov_b32 exec_lo, 0\n"
+                   "global_load_async_to_lds_b8 v0, v[0:1], off\n"
+                   "s_mov_b32 exec_lo, s7"
+                   ::: "v0", "v1", "s7", "memory");
+}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..3a1549c3b325e3
--- /dev/null
+++ b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
@@ -0,0 +1,15 @@
+// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
+// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
+// RUN: cat %s >> %t.s
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index 218d2dffa302ce..8a060a0118ae60 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17222,6 +17222,34 @@ It supports AMDGCN GFX6-GFX11.
 
 This section describes general syntax for instructions and operands.
 
+(amdgpu-async-barrier-replay-warning)=
+
+#### Async Barrier Arrival Warning
+
+The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
+unless XNACK is explicitly disabled. The assembler checks hand-written and
+inline assembly for the same sequence. This check is also omitted when XNACK
+is explicitly disabled.
+
+The assembler warns by default when it cannot recognize the compiler's dummy
+form: `s_mov_b32 exec_lo, 0` followed by
+`global_load_async_to_lds_b8` with a vector address, issued after an arrival
+and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
+not change EXEC, may intervene between the EXEC clear and the load.
+Other load opcodes, scalar-address forms,
+and active loads do not suppress the warning. Recognition stops at labels,
+control-flow instructions, and the end of assembly. Assembler directives
+can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
+block is checked separately. A warning reports missing recognized protection;
+absence of a warning does not prove control-flow safety, correct EXEC
+preservation, or correct async wait counts.
+
+Disable only this check in Clang with
+`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
+`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
+suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
+errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
+
 #### Instructions
 
 An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 73e0ad143d7008..e80796200ea92a 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,12 @@ Makes programs 10x faster by doing Special New Thing.
 
 ### Changes to the AMDGPU Backend
 
+* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
+  assembler warnings for unrecognized sequences in handwritten or inline
+  assembly. Both are omitted when XNACK is explicitly disabled. See
+  {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
+  warning controls.
+
 * Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
   and `amdgpu.sramecc` module flags.
 * `llvm.amdgcn.make.buffer.rsrc` now accepts any integer width for its
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 091279fc95fff2..231764b7381153 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
 #include "Utils/AMDKernelCodeTUtils.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
@@ -40,10 +41,12 @@
 #include "llvm/Support/AMDGPUObjLinkingInfo.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 #include <optional>
+#include <utility>
 
 using namespace llvm;
 using namespace llvm::AMDGPU;
@@ -51,6 +54,12 @@ using namespace llvm::amdhsa;
 
 namespace {
 
+static cl::opt<bool> WarnAsyncBarrierReplay(
+    "amdgpu-asm-warn-async-barrier-replay",
+    cl::desc("Warn about async barrier arrivals without a recognized replay "
+             "workaround in assembly"),
+    cl::init(true), cl::Hidden);
+
 class AMDGPUAsmParser;
 
 enum RegisterKind {
@@ -1135,6 +1144,12 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   const AMDGPU::GPUKind Gfx;
   const AMDGPU::IsaVersion ISA;
 
+  SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
+  bool AsyncBarrierExecCleared = false;
+
+  void checkAsyncBarrierReplay(const MCInst &Inst);
+  void warnPendingAsyncBarrierArrivals();
+
   /// @name Auto-generated Match Functions
   /// {
 
@@ -5839,6 +5854,64 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
   return false;
 }
 
+void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
+  for (SMLoc Loc : PendingAsyncBarrierArrivals) {
+    Warning(Loc, "no recognized replay workaround after "
+                 "'ds_atomic_async_barrier_arrive_b64' on this GPU");
+  }
+  PendingAsyncBarrierArrivals.clear();
+  AsyncBarrierExecCleared = false;
+}
+
+void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
+  if (!WarnAsyncBarrierReplay ||
+      !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
+      getTargetStreamer().getTargetID()->getXnackSetting() ==
+          AMDGPU::TargetIDSetting::Off)
+    return;
+
+  const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
+  switch (Inst.getOpcode()) {
+  case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
+    PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
+    return;
+  // These instructions change control flow but their MC descriptors do not
+  // carry branch, return, or trap flags.
+  case AMDGPU::S_RFE_B64_gfx1250:
+  case AMDGPU::S_TRAP_gfx12:
+    warnPendingAsyncBarrierArrivals();
+    return;
+  case AMDGPU::S_MOV_B32_gfx12:
+    if (!PendingAsyncBarrierArrivals.empty()) {
+      const MCOperand &Dst = Inst.getOperand(0);
+      const MCOperand &Src = Inst.getOperand(1);
+      AsyncBarrierExecCleared = Dst.isReg() &&
+                                mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
+                                Src.isImm() && Src.getImm() == 0;
+    }
+    return;
+  case AMDGPU::S_SET_VGPR_MSB_gfx12:
+    // VGPR encoding can insert this between clearing EXEC and the dummy load.
+    AsyncBarrierExecCleared = ExecCleared;
+    return;
+  case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
+    // Recognize only the compiler's EXEC-zero B8 dummy form.
+    if (ExecCleared)
+      PendingAsyncBarrierArrivals.clear();
+    return;
+  }
+
+  if (PendingAsyncBarrierArrivals.empty())
+    return;
+
+  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
+  // Check only a straight-line sequence. Labels and control-flow changes end
+  // recognition rather than allowing a load on another path to hide a warning.
+  if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
+      (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
+    warnPendingAsyncBarrierArrivals();
+}
+
 bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                               OperandVector &Operands,
                                               MCStreamer &Out,
@@ -5878,6 +5951,7 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       return true;
     }
     emitTargetDirective();
+    checkAsyncBarrierReplay(Inst);
     Out.emitInstruction(Inst, getSTI());
     // Record for kernel prologue checking.
     OpcodeStream.push_back(Inst.getOpcode());
@@ -7022,6 +7096,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
 }
 
 void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
+  warnPendingAsyncBarrierArrivals();
   // Record every parsed label in the timeline so that, at end of file, the
   // instructions following a kernel's label can be located regardless of
   // whether the .amdhsa_kernel directive came before or after the label.
@@ -7052,6 +7127,7 @@ void AMDGPUAsmParser::checkKernelPrologues() {
 }
 
 void AMDGPUAsmParser::onEndOfFile() {
+  warnPendingAsyncBarrierArrivals();
   emitTargetDirective();
   checkKernelPrologues();
   if (InfoData)
@@ -7061,6 +7137,11 @@ void AMDGPUAsmParser::onEndOfFile() {
 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getString();
 
+  // Debug locations do not interrupt the instruction sequence. Other
+  // directives may change sections or emit bytes outside this parser.
+  if (IDVal != ".loc" && IDVal != ".file")
+    warnPendingAsyncBarrierArrivals();
+
   if (isHsaAbi(getSTI())) {
     if (IDVal == ".amdhsa_kernel")
       return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index 313f67ea31e22f..cc8ed4cdd251f3 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -265,6 +265,12 @@ def FeaturePreciseMemory
     : SubtargetFeature<"precise-memory", "EnablePreciseMemory",
                        "true", "Enable precise memory mode">;
 
+defm AsyncBarrierArriveReplayBug : AMDGPUSubtargetFeature<
+  "async-barrier-arrive-replay-bug",
+  "Enable the async barrier arrival workaround",
+  /*GenPredicate=*/0, /*GenAssemblerPredicate=*/0, [], InlineIgnore
+>;
+
 defm SGPRInitBug : AMDGPUSubtargetFeature<"sgpr-init-bug",
   "VI SGPR initialization bug requiring a fixed SGPR allocation size",
   /*GenPredicate=*/1, /*GenAssemblerPredicate=*/1, [], InlineIgnore
@@ -2493,6 +2499,7 @@ def FeatureISAVersion12 : FeatureSet<
 
 def FeatureISAVersion12_50_Common : FeatureSet<
   [FeatureGFX12,
+   FeatureAsyncBarrierArriveReplayBug,
    FeatureGFX1250Insts,
    FeatureBackOffBarrier,
    FeatureRequiresAlignedVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 0356f9aaa822e1..b2d253d67bf793 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -345,6 +345,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool isXNACKEnabled() const { return TargetID.isXnackOnOrAny(); }
 
+  bool needsAsyncBarrierArriveReplayWorkaround() const {
+    // Generic 12.5 code has no explicit XNACK setting; keep the workaround.
+    return hasAsyncBarrierArriveReplayBug() &&
+           TargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Off;
+  }
+
   bool hasRelaxedBufferOOBMode() const { return BufferOOBRelaxed; }
   bool hasRelaxedTBufferOOBMode() const { return TBufferOOBRelaxed; }
 
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index cb51eeb8b13c96..ce761f94b97a2f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1859,7 +1859,20 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
       // If found any unused scratch SGPR, reserve the register itself for Exec
       // copy and there is no need for any spills in that case.
       MFI->setSGPRForEXECCopy(UnusedScratchReg);
-      MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
+      // Only rewrite EXEC saves and restores. Other uses, such as inline
+      // assembly operands and their callee saves, name fixed registers.
+      const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+      for (MachineOperand &Op :
+           make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
+        MachineInstr &MI = *Op.getParent();
+        if (MI.getOpcode() == LMC.OrSaveExecOpc ||
+            (MI.getOpcode() == LMC.MovOpc &&
+             (MI.getOperand(0).getReg() == LMC.ExecReg ||
+              (MI.getOperand(1).isReg() &&
+               MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+          Op.setReg(UnusedScratchReg);
+        }
+      }
       LiveUnits.addReg(UnusedScratchReg);
     } else {
       // Needs spill.
@@ -1933,19 +1946,23 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
   const SIInstrInfo *TII = ST.getInstrInfo();
   bool NeedExecCopyReservedReg = false;
+  bool NeedExecCopyForAsyncBarrier = false;
 
   MachineInstr *ReturnMI = nullptr;
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
       // TODO: Walking through all MBBs here would be a bad heuristic. Better
       // handle them elsewhere.
-      if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+      if (TII->isWWMRegSpillOpcode(MI.getOpcode())) {
         NeedExecCopyReservedReg = true;
-      else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
-               MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
-               MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
-               (MFI->isChainFunction() &&
-                TII->isChainCallOpcode(MI.getOpcode()))) {
+      } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
+                 ST.needsAsyncBarrierArriveReplayWorkaround()) {
+        NeedExecCopyForAsyncBarrier = true;
+      } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
+                 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
+                 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
+                 (MFI->isChainFunction() &&
+                  TII->isChainCallOpcode(MI.getOpcode()))) {
         // We expect all return to be the same size.
         assert(!ReturnMI ||
                (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
@@ -2013,7 +2030,10 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   if (!ST.hasGFX90AInsts())
     SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
 
-  determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
+  // Whole-wave spills use the original EXEC mask from the first argument, but
+  // the barrier workaround needs separate storage for the current EXEC mask.
+  determinePrologEpilogSGPRSaves(
+      MF, SavedVGPRs, NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
 
   // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
   // allow the default insertion to handle them.
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 2db7fc52495448..3986b87de2a4a6 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 
 using namespace llvm;
@@ -3497,6 +3498,43 @@ bool SIInsertWaitcnts::run() {
 
   bool Modified = false;
 
+  if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
+    // Collect arrivals before inserting new definitions of ASYNCcnt.
+    SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+    for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+      if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+        AsyncBarrierArrivals.push_back(&MI);
+    }
+
+    if (!AsyncBarrierArrivals.empty()) {
+      const Register SavedExec = MFI->getSGPRForEXECCopy();
+      if (!SavedExec) {
+        report_fatal_error(
+            "missing EXEC copy SGPR for async barrier workaround");
+      }
+
+      for (MachineInstr *MI : AsyncBarrierArrivals) {
+        MachineBasicBlock &MBB = *MI->getParent();
+        const DebugLoc &DL = MI->getDebugLoc();
+        // Insert after scheduling so the dummy stays after its arrival.
+        auto InsertPt = std::next(MI->getIterator());
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+            .addReg(AMDGPU::EXEC_LO);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addImm(0);
+        // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+            .addReg(AMDGPU::VGPR0, RegState::Undef)
+            .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+            .addImm(0)
+            .addImm(0);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addReg(SavedExec);
+        Modified = true;
+      }
+    }
+  }
+
   MachineBasicBlock &EntryBB = MF.front();
 
   if (!MFI->isEntryFunction() &&
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index d0a57e163d29a0..f48e2194242a42 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -619,9 +619,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
     if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
                              TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
       FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
-  } else {
-    // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
-    // spills/copies. Reset the SGPR reserved for EXEC copy.
+  } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
+             none_of(MRI.def_instructions(AMDGPU::ASYNCcnt),
+                     [](const MachineInstr &MI) {
+                       return MI.getOpcode() ==
+                              AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
+                     })) {
+    // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
+    // needs the reserved register for EXEC copy.
     FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
new file mode 100644
index 00000000000000..155f0d3b296a3a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
@@ -0,0 +1,32 @@
+# NOTE: Do not autogenerate. Check the bank switch inside the workaround and assemble compiler output with fatal warnings.
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
+
+--- |
+  define amdgpu_cs void @high_vgpr_arrive() { ret void }
+...
+
+---
+name: high_vgpr_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr256
+    ; The arrival needs a high VGPR, but the dummy uses v[0:1]. The late
+    ; encoding pass resets the bank after EXEC is cleared.
+    ; CHECK-LABEL: high_vgpr_arrive:
+    ; CHECK: s_set_vgpr_msb 1
+    ; CHECK-NEXT: ds_atomic_async_barrier_arrive_b64 v0 /*v256*/
+    ; CHECK: s_mov_b32 s105, exec_lo
+    ; CHECK-NEXT: s_mov_b32 exec_lo, 0
+    ; CHECK-NEXT: s_set_vgpr_msb 0x100
+    ; CHECK-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+    ; CHECK-NEXT: s_mov_b32 exec_lo, s105
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr256, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
new file mode 100644
index 00000000000000..5998e619fbcab7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll
@@ -0,0 +1,494 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,SDAGOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -verify-machineinstrs < %s | FileCheck --check-prefixes=ON,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdpal -amdgpu-xnack=0 -verify-machineinstrs < %s | FileCheck --check-prefixes=OFF,GISELOFF %s
+
+; The caller initializes the LDS barrier at offset zero. Only the active-mask
+; argument is used, since ordinary arguments are poison in inactive lanes.
+; gfx1250 and gfx1251 do not support wave64.
+
+; The barrier must use a separate scratch SGPR from the saved incoming EXEC,
+; even when the active argument has no explicit uses.
+define amdgpu_gfx_whole_wave void @whole_wave_barrier(i1 %active) {
+; ON-LABEL: whole_wave_barrier:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mov_b32_e32 v0, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_barrier:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mov_b32_e32 v0, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret void
+}
+
+; Moving the EXEC scratch must preserve fixed-register assembly operands and the
+; corresponding callee saves, in both whole-wave and ordinary functions.
+define amdgpu_gfx_whole_wave i32 @whole_wave_clobber_exec_scratch(i1 %active) {
+; ON-LABEL: whole_wave_clobber_exec_scratch:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT:    scratch_store_b32 off, v2, s32 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v1, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_writelane_b32 v2, s105, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    s_mov_b32 s105, 0
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT:    v_readlane_b32 s105, v2, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT:    scratch_load_b32 v2, off, s32 nv
+; ON-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; ON-NEXT:    scratch_load_b32 v1, off, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_clobber_exec_scratch:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v1, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_writelane_b32 v2, s105, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    s_mov_b32 s105, 0
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT:    v_readlane_b32 s105, v2, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 nv
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; OFF-NEXT:    scratch_load_b32 v1, off, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret i32 %value
+}
+
+define i32 @callable_clobber_exec_scratch() {
+; ON-LABEL: callable_clobber_exec_scratch:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    v_writelane_b32 v2, s105, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    s_mov_b32 s105, 0
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; ON-NEXT:    v_readlane_b32 s105, v2, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s0, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: callable_clobber_exec_scratch:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    v_writelane_b32 v2, s105, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    s_mov_b32 s105, 0
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s105
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; OFF-NEXT:    v_readlane_b32 s105, v2, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v1
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %value = call i32 asm sideeffect "s_mov_b32 s105, 0", "={s105}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret i32 %value
+}
+
+; Keep the active argument live across the barrier and exercise SGPR and VGPR
+; callee saves as well as the reserved scratch register.
+define amdgpu_gfx_whole_wave i32 @whole_wave_active(i1 %active) {
+; SDAG-LABEL: whole_wave_active:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT:    s_wait_kmcnt 0x0
+; SDAG-NEXT:    s_xor_saveexec_b32 s0, -1
+; SDAG-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; SDAG-NEXT:    scratch_store_b32 off, v1, s32 nv
+; SDAG-NEXT:    ; meta instruction
+; SDAG-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_mov_b32 exec_lo, -1
+; SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAG-NEXT:    v_writelane_b32 v1, s68, 0
+; SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    ;;#ASMSTART
+; SDAG-NEXT:    ; clobber CSR
+; SDAG-NEXT:    ;;#ASMEND
+; SDAG-NEXT:    s_and_b32 s1, s0, exec_lo
+; SDAG-NEXT:    s_cselect_b32 s1, 42, 7
+; SDAG-NEXT:    v_readlane_b32 s68, v1, 0
+; SDAG-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; SDAG-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAG-NEXT:    s_mov_b32 s2, exec_lo
+; SDAG-NEXT:    s_mov_b32 exec_lo, 0
+; SDAG-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; SDAG-NEXT:    s_mov_b32 exec_lo, s2
+; SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAG-NEXT:    v_mov_b32_e32 v0, s1
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_xor_b32 exec_lo, s0, -1
+; SDAG-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; SDAG-NEXT:    scratch_load_b32 v1, off, s32 nv
+; SDAG-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAG-NEXT:    s_wait_xcnt 0x0
+; SDAG-NEXT:    s_mov_b32 exec_lo, s0
+; SDAG-NEXT:    s_wait_loadcnt 0x0
+; SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; SDAGOFF-LABEL: whole_wave_active:
+; SDAGOFF:       ; %bb.0:
+; SDAGOFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; SDAGOFF-NEXT:    s_wait_kmcnt 0x0
+; SDAGOFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; SDAGOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; SDAGOFF-NEXT:    scratch_store_b32 off, v1, s32 nv
+; SDAGOFF-NEXT:    ; meta instruction
+; SDAGOFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; SDAGOFF-NEXT:    s_mov_b32 exec_lo, -1
+; SDAGOFF-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; SDAGOFF-NEXT:    v_writelane_b32 v1, s68, 0
+; SDAGOFF-NEXT:    v_mov_b32_e32 v0, 0
+; SDAGOFF-NEXT:    ;;#ASMSTART
+; SDAGOFF-NEXT:    ; clobber CSR
+; SDAGOFF-NEXT:    ;;#ASMEND
+; SDAGOFF-NEXT:    s_and_b32 s1, s0, exec_lo
+; SDAGOFF-NEXT:    s_cselect_b32 s1, 42, 7
+; SDAGOFF-NEXT:    v_readlane_b32 s68, v1, 0
+; SDAGOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; SDAGOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; SDAGOFF-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; SDAGOFF-NEXT:    v_mov_b32_e32 v0, s1
+; SDAGOFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; SDAGOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; SDAGOFF-NEXT:    scratch_load_b32 v1, off, s32 nv
+; SDAGOFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; SDAGOFF-NEXT:    s_mov_b32 exec_lo, s0
+; SDAGOFF-NEXT:    s_wait_loadcnt 0x0
+; SDAGOFF-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: whole_wave_active:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_kmcnt 0x0
+; GISEL-NEXT:    s_xor_saveexec_b32 s0, -1
+; GISEL-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; GISEL-NEXT:    scratch_store_b32 off, v1, s32 nv
+; GISEL-NEXT:    ; meta instruction
+; GISEL-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_mov_b32 exec_lo, -1
+; GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISEL-NEXT:    v_writelane_b32 v1, s68, 0
+; GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    ;;#ASMSTART
+; GISEL-NEXT:    ; clobber CSR
+; GISEL-NEXT:    ;;#ASMEND
+; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISEL-NEXT:    v_readlane_b32 s68, v1, 0
+; GISEL-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; GISEL-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISEL-NEXT:    s_mov_b32 s1, exec_lo
+; GISEL-NEXT:    s_mov_b32 exec_lo, 0
+; GISEL-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_xor_b32 exec_lo, s0, -1
+; GISEL-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; GISEL-NEXT:    scratch_load_b32 v1, off, s32 nv
+; GISEL-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; GISEL-NEXT:    s_wait_xcnt 0x0
+; GISEL-NEXT:    s_mov_b32 exec_lo, s0
+; GISEL-NEXT:    s_wait_loadcnt 0x0
+; GISEL-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GISELOFF-LABEL: whole_wave_active:
+; GISELOFF:       ; %bb.0:
+; GISELOFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISELOFF-NEXT:    s_wait_kmcnt 0x0
+; GISELOFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; GISELOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Spill
+; GISELOFF-NEXT:    scratch_store_b32 off, v1, s32 nv
+; GISELOFF-NEXT:    ; meta instruction
+; GISELOFF-NEXT:    scratch_store_b32 off, v0, s32 offset:4 nv
+; GISELOFF-NEXT:    s_mov_b32 exec_lo, -1
+; GISELOFF-NEXT:    scratch_store_b32 off, v40, s32 offset:8 nv ; 4-byte Folded Spill
+; GISELOFF-NEXT:    v_writelane_b32 v1, s68, 0
+; GISELOFF-NEXT:    v_mov_b32_e32 v0, 0
+; GISELOFF-NEXT:    ;;#ASMSTART
+; GISELOFF-NEXT:    ; clobber CSR
+; GISELOFF-NEXT:    ;;#ASMEND
+; GISELOFF-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GISELOFF-NEXT:    v_readlane_b32 s68, v1, 0
+; GISELOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; GISELOFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GISELOFF-NEXT:    scratch_load_b32 v40, off, s32 offset:8 nv ; 4-byte Folded Reload
+; GISELOFF-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; GISELOFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; GISELOFF-NEXT:    s_clause 0x1 ; 8-byte Folded Reload
+; GISELOFF-NEXT:    scratch_load_b32 v1, off, s32 nv
+; GISELOFF-NEXT:    scratch_load_b32 v0, off, s32 offset:4 nv
+; GISELOFF-NEXT:    s_mov_b32 exec_lo, s0
+; GISELOFF-NEXT:    s_wait_loadcnt 0x0
+; GISELOFF-NEXT:    s_set_pc_i64 s[30:31]
+  call void asm sideeffect "; clobber CSR", "~{v40},~{s68}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  %result = select i1 %active, i32 42, i32 7
+  ret i32 %result
+}
+
+; A divergent region has a current EXEC mask distinct from the original
+; active argument. Preserve both through the workaround.
+define amdgpu_gfx_whole_wave i32 @whole_wave_divergent(i1 %active) {
+; ON-LABEL: whole_wave_divergent:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mbcnt_lo_u32_b32 v0, -1, 0
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; ON-NEXT:    v_and_b32_e32 v0, 1, v0
+; ON-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; ON-NEXT:    s_cbranch_execz .LBB4_2
+; ON-NEXT:  ; %bb.1: ; %arrive
+; ON-NEXT:    v_mov_b32_e32 v0, 0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s2, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s2
+; ON-NEXT:  .LBB4_2: ; %exit
+; ON-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; ON-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; ON-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_divergent:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv ; 4-byte Folded Spill
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mbcnt_lo_u32_b32 v0, -1, 0
+; OFF-NEXT:    s_mov_b32 s1, exec_lo
+; OFF-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; OFF-NEXT:    v_and_b32_e32 v0, 1, v0
+; OFF-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; OFF-NEXT:  ; %bb.1: ; %arrive
+; OFF-NEXT:    v_mov_b32_e32 v0, 0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:  ; %bb.2: ; %exit
+; OFF-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; OFF-NEXT:    v_cndmask_b32_e64 v0, 7, 42, s0
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %parity = and i32 %lane, 1
+  %cond = icmp eq i32 %parity, 0
+  br i1 %cond, label %arrive, label %exit
+
+arrive:
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  br label %exit
+
+exit:
+  %result = select i1 %active, i32 42, i32 7
+  ret i32 %result
+}
+
+; Keep both dummy-load address registers live across the workaround.
+define amdgpu_gfx_whole_wave void @whole_wave_live_vgprs(i1 %active) {
+; ON-LABEL: whole_wave_live_vgprs:
+; ON:       ; %bb.0:
+; ON-NEXT:    s_wait_loadcnt_dscnt 0x0
+; ON-NEXT:    s_wait_kmcnt 0x0
+; ON-NEXT:    s_xor_saveexec_b32 s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; ON-NEXT:    scratch_store_b32 off, v0, s32 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v1, s32 offset:4 nv
+; ON-NEXT:    ; meta instruction
+; ON-NEXT:    scratch_store_b32 off, v2, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, -1
+; ON-NEXT:    v_mov_b32_e32 v2, 0
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    v_mov_b32_e32 v0, 0x7b
+; ON-NEXT:    v_mov_b32_e32 v1, 0x1c8
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
+; ON-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; ON-NEXT:    s_mov_b32 s1, exec_lo
+; ON-NEXT:    s_mov_b32 exec_lo, 0
+; ON-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; ON-NEXT:    s_mov_b32 exec_lo, s1
+; ON-NEXT:    ;;#ASMSTART
+; ON-NEXT:    ; use v[0:1]
+; ON-NEXT:    ;;#ASMEND
+; ON-NEXT:    s_xor_b32 exec_lo, s0, -1
+; ON-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; ON-NEXT:    scratch_load_b32 v0, off, s32 nv
+; ON-NEXT:    scratch_load_b32 v1, off, s32 offset:4 nv
+; ON-NEXT:    scratch_load_b32 v2, off, s32 offset:8 nv
+; ON-NEXT:    s_wait_xcnt 0x0
+; ON-NEXT:    s_mov_b32 exec_lo, s0
+; ON-NEXT:    s_wait_loadcnt 0x0
+; ON-NEXT:    s_set_pc_i64 s[30:31]
+;
+; OFF-LABEL: whole_wave_live_vgprs:
+; OFF:       ; %bb.0:
+; OFF-NEXT:    s_wait_loadcnt_dscnt 0x0
+; OFF-NEXT:    s_wait_kmcnt 0x0
+; OFF-NEXT:    s_xor_saveexec_b32 s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Spill
+; OFF-NEXT:    scratch_store_b32 off, v0, s32 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v1, s32 offset:4 nv
+; OFF-NEXT:    ; meta instruction
+; OFF-NEXT:    scratch_store_b32 off, v2, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, -1
+; OFF-NEXT:    v_mov_b32_e32 v2, 0
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    v_mov_b32_e32 v0, 0x7b
+; OFF-NEXT:    v_mov_b32_e32 v1, 0x1c8
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
+; OFF-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; OFF-NEXT:    ;;#ASMSTART
+; OFF-NEXT:    ; use v[0:1]
+; OFF-NEXT:    ;;#ASMEND
+; OFF-NEXT:    s_xor_b32 exec_lo, s0, -1
+; OFF-NEXT:    s_clause 0x2 ; 12-byte Folded Reload
+; OFF-NEXT:    scratch_load_b32 v0, off, s32 nv
+; OFF-NEXT:    scratch_load_b32 v1, off, s32 offset:4 nv
+; OFF-NEXT:    scratch_load_b32 v2, off, s32 offset:8 nv
+; OFF-NEXT:    s_mov_b32 exec_lo, s0
+; OFF-NEXT:    s_wait_loadcnt 0x0
+; OFF-NEXT:    s_set_pc_i64 s[30:31]
+  %values = call <2 x i32> asm sideeffect "v_mov_b32 v0, 123\0Av_mov_b32 v1, 456", "={v[0:1]}"()
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  call void asm sideeffect "; use $0", "{v[0:1]}"(<2 x i32> %values)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
new file mode 100644
index 00000000000000..6158c7ec97427e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
@@ -0,0 +1,46 @@
+; NOTE: Do not autogenerate. Check workaround ordering on affected targets and absence on unaffected targets.
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu13.10-amd-amdhsa -mattr=+lds-barrier-arrive-atomic < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=0 < %s | FileCheck --check-prefix=UNAFFECTED %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50s-amd-amdhsa -amdgpu-xnack=1 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.5-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-async-barrier-arrive-replay-bug < %s | FileCheck --check-prefix=UNAFFECTED %s
+
+; Check an LDS barrier arrival followed by a global load.
+ at bar = internal addrspace(3) global i64 undef, align 8
+
+; AFFECTED-LABEL: replay_window:
+; AFFECTED: ds_atomic_async_barrier_arrive_b64
+; AFFECTED-NOT: global_load_b32
+; AFFECTED: s_mov_b32 [[SAVED:s[0-9]+]], exec_lo
+; AFFECTED-NEXT: s_mov_b32 exec_lo, 0
+; AFFECTED-NEXT: global_load_async_to_lds_b8
+; AFFECTED-NEXT: s_mov_b32 exec_lo, [[SAVED]]
+; AFFECTED: global_load_b32
+;
+; UNAFFECTED-LABEL: replay_window:
+; UNAFFECTED: ds_atomic_async_barrier_arrive_b64
+; UNAFFECTED-NOT: global_load_async_to_lds
+; UNAFFECTED: global_load_b32
+define amdgpu_kernel void @replay_window(ptr addrspace(1) %src,
+                                         ptr addrspace(1) %dst) {
+entry:
+  store i64 0, ptr addrspace(3) @bar, align 8
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) @bar)
+  %value = load volatile i32, ptr addrspace(1) %src, align 4
+  store volatile i32 %value, ptr addrspace(1) %dst, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
new file mode 100644
index 00000000000000..09eb65e1ac5cd8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
@@ -0,0 +1,269 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+
+--- |
+  define amdgpu_kernel void @live_scc() { ret void }
+  define amdgpu_kernel void @empty_exec() { ret void }
+  define amdgpu_kernel void @no_arrive() { ret void }
+  define amdgpu_kernel void @multiple_arrivals() { ret void }
+  define amdgpu_kernel void @partial_wait() { ret void }
+  define amdgpu_kernel void @expert_partial_wait() #0 { ret void }
+  attributes #0 = { "amdgpu-expert-scheduling-mode"="true" }
+...
+
+---
+name: live_scc
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; ON-LABEL: name: live_scc
+    ; ON: liveins: $sgpr0, $vgpr0
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: live_scc
+    ; OFF: liveins: $sgpr0, $vgpr0
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    ; OFF-NEXT: S_ENDPGM 0
+    S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
+    S_ENDPGM 0
+...
+
+---
+name: empty_exec
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; ON-LABEL: name: empty_exec
+    ; ON: liveins: $vgpr0
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: $exec_lo = S_MOV_B32 -1
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: empty_exec
+    ; OFF: liveins: $vgpr0
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: $exec_lo = S_MOV_B32 0
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: $exec_lo = S_MOV_B32 -1
+    ; OFF-NEXT: S_ENDPGM 0
+    $exec_lo = S_MOV_B32 0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $exec_lo = S_MOV_B32 -1
+    S_ENDPGM 0
+...
+
+---
+name: no_arrive
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    ; ON-LABEL: name: no_arrive
+    ; ON: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: no_arrive
+    ; OFF: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    S_ENDPGM 0
+...
+
+---
+name: multiple_arrivals
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  ; ON-LABEL: name: multiple_arrivals
+  ; ON: bb.0:
+  ; ON-NEXT:   successors: %bb.1(0x80000000)
+  ; ON-NEXT:   liveins: $vgpr0
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT:   S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+  ; ON-NEXT:   $sgpr64_sgpr65 = S_MOV_B64 0
+  ; ON-NEXT:   V_NOP_e32 implicit $exec
+  ; ON-NEXT:   GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+  ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   S_BRANCH %bb.1
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT: bb.1:
+  ; ON-NEXT:   liveins: $vgpr0
+  ; ON-NEXT: {{  $}}
+  ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   S_ENDPGM 0
+  ;
+  ; OFF-LABEL: name: multiple_arrivals
+  ; OFF: bb.0:
+  ; OFF-NEXT:   successors: %bb.1(0x80000000)
+  ; OFF-NEXT:   liveins: $vgpr0
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT:   $sgpr64_sgpr65 = S_MOV_B64 0
+  ; OFF-NEXT:   V_NOP_e32 implicit $exec
+  ; OFF-NEXT:   GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+  ; OFF-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; OFF-NEXT:   S_BRANCH %bb.1
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT: bb.1:
+  ; OFF-NEXT:   liveins: $vgpr0
+  ; OFF-NEXT: {{  $}}
+  ; OFF-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+  ; OFF-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    liveins: $vgpr0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name: partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; The extra async operation must not relax the user's partial wait.
+    ; ON-LABEL: name: partial_wait
+    ; ON: liveins: $vgpr0, $vgpr1
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: partial_wait
+    ; OFF: liveins: $vgpr0, $vgpr1
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    S_ENDPGM 0
+...
+
+---
+name: expert_partial_wait
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; The extra async operation must not relax the user's partial wait.
+    ; ON-LABEL: name: expert_partial_wait
+    ; ON: liveins: $vgpr0, $vgpr1
+    ; ON-NEXT: {{  $}}
+    ; ON-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
+    ; ON-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; ON-NEXT: V_NOP_e32 implicit $exec
+    ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; ON-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
+    ; ON-NEXT: $exec_lo = S_MOV_B32 0
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; ON-NEXT: S_ENDPGM 0
+    ;
+    ; OFF-LABEL: name: expert_partial_wait
+    ; OFF: liveins: $vgpr0, $vgpr1
+    ; OFF-NEXT: {{  $}}
+    ; OFF-NEXT: $sgpr64_sgpr65 = S_MOV_B64 0
+    ; OFF-NEXT: V_NOP_e32 implicit $exec
+    ; OFF-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
+    ; OFF-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+    ; OFF-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    ; OFF-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; OFF-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    ; OFF-NEXT: S_ENDPGM 0
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
new file mode 100644
index 00000000000000..0eb8ad252cb4e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
@@ -0,0 +1,24 @@
+; NOTE: Do not autogenerate. Object emission invokes the inline assembler; null emission does not. Check warning presence and absence.
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -filetype=obj -o %t.o < %s 2>&1 | count 0
+; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+
+define amdgpu_kernel void @unprotected_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+  ret void
+}
+
+define amdgpu_kernel void @protected_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0\0As_mov_b32 s7, exec_lo\0As_mov_b32 exec_lo, 0\0Aglobal_load_async_to_lds_b8 v0, v[0:1], off\0As_mov_b32 exec_lo, s7", "~{v0},~{v1},~{s7},~{memory}"()
+  ret void
+}
+
+; A later inline-asm block cannot protect an arrival in an earlier block.
+define amdgpu_kernel void @split_inline_asm() {
+  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
+; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+  call void asm sideeffect "global_load_async_to_lds_b32 v0, v[0:1], off", "~{v0},~{v1},~{memory}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
index c42d8b4c9cc143..168a97ebd53302 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll
@@ -85,6 +85,10 @@ define void @taildup_ds_atomic_async_barrier_arrive(ptr addrspace(1) %a, ptr add
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v2
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s1, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s1
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   br i1 %cond, label %bb1, label %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
index 791a2a3a767eb2..59951e39cb06ed 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -12,6 +12,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64(ptr addrspace(3) %bar) {
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s0, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s0
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
@@ -26,6 +30,10 @@ define void @test_ds_atomic_async_barrier_arrive_b64_off(ptr addrspace(3) %in) {
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0 offset:8184
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s0, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s0
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
 entry:
   %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
@@ -44,6 +52,10 @@ define amdgpu_ps void @test_ds_atomic_async_barrier_arrive_b64_s(ptr addrspace(3
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
 ; GCN-NEXT:    ds_atomic_async_barrier_arrive_b64 v0
 ; GCN-NEXT:    s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT:    s_mov_b32 s105, exec_lo
+; GCN-NEXT:    s_mov_b32 exec_lo, 0
+; GCN-NEXT:    global_load_async_to_lds_b8 v0, v[0:1], off
+; GCN-NEXT:    s_mov_b32 exec_lo, s105
 ; GCN-NEXT:    s_endpgm
 entry:
   call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
new file mode 100644
index 00000000000000..8a8bff7573ade5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
@@ -0,0 +1,38 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across the EXEC-clear sequence.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+
+// VGPR bank changes preserve EXEC across the compiler's dummy-load sequence.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// A bank change alone does not establish that EXEC is clear.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// A bank change does not hide a subsequent write to EXEC.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_set_vgpr_msb 0x100
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
+
+// Other intervening instructions still break the pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+s_set_vgpr_msb 0x100
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
new file mode 100644
index 00000000000000..f1931aebaef82e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
@@ -0,0 +1,27 @@
+// NOTE: Do not autogenerate. Check target-ID XNACK settings and warning locations.
+// RUN: split-file %s %t
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/on.s 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/off.s 2>&1 | count 0
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/on.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
+// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/off.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50s-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.5-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
+
+// ON: <stdin>:2:1: warning: no recognized replay workaround
+// ANY: <stdin>:1:1: warning: no recognized replay workaround
+
+//--- on.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack+"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- off.s
+.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
+
+//--- any.s
+ds_atomic_async_barrier_arrive_b64 v0
+s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..719e87544c98a9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
@@ -0,0 +1,187 @@
+// NOTE: Do not autogenerate. Check diagnostic locations and absence across sequence boundaries.
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | not llvm-mc -triple=amdgpu12.50 -filetype=null --fatal-warnings 2>&1 | FileCheck %s --check-prefix=FATAL --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=true < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null --no-warn < %s 2>&1 | count 0
+// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | llvm-mc -triple=amdgpu13.10 -mattr=+lds-barrier-arrive-atomic -filetype=null 2>&1 | count 0
+// RUN: llvm-mc -triple=amdgpu12.50 -mattr=-async-barrier-arrive-replay-bug -filetype=null < %s 2>&1 | count 0
+// FATAL: <stdin>:1:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+
+// Non-async VMEM ends recognition before the following load.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_nop 0
+global_load_b32 v1, v[2:3], off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// The compiler's dummy-load sequence is recognized without fixing its SGPR.
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s7, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v0, v[0:1], off
+s_mov_b32 exec_lo, s7
+s_endpgm
+
+// Active B8 loads and scalar-address forms are not the recognized dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+global_load_async_to_lds_b8 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+
+// B32 forms are not recognized, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v5, s[2:3]
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b32 v4, v[2:3], off
+s_endpgm
+
+// Cluster loads do not suppress the warning, even with EXEC cleared.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 m0, 0
+s_mov_b32 exec_lo, 0
+cluster_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Clearing another register or using a nonzero EXEC is not the dummy form.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 s7, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 1
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Intervening instructions break the immediately preceding EXEC-clear pattern.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+s_nop 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_endpgm
+
+// Source locations do not interrupt a protective sequence.
+.file 1 "async-barrier.s"
+ds_atomic_async_barrier_arrive_b64 v0
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+.loc 1 10 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// One qualifying load can follow several arrivals.
+ds_atomic_async_barrier_arrive_b64 v0
+ds_atomic_async_barrier_arrive_b64 v1
+s_mov_b32 s9, exec_lo
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+s_mov_b32 exec_lo, s9
+s_endpgm
+
+// B64/B128 forms are not recognized.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b64 v4, v[2:3], off
+global_load_b32 v1, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b128 v4, v[2:3], off
+s_endpgm
+
+// Recognize scratch, flat and buffer VMEM as well as global VMEM.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+scratch_load_b32 v1, v2, off
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+flat_load_b32 v1, v[2:3]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+buffer_load_b32 v1, v2, s[4:7], s8 offen
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A load beyond a branch or label does not prove protection on every path.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_branch next
+next:
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_mov_b32 exec_lo, 0
+another_path:
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Calls, indirect branches, traps and exception returns end recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_swap_pc_i64 s[2:3], s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_set_pc_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_trap 2
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_rfe_i64 s[0:1]
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// A conditional branch must not let a fall-through load hide the warning.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+s_cbranch_scc0 1
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Bytes emitted by a directive interrupt recognition.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
+.long 0
+s_mov_b32 exec_lo, 0
+global_load_async_to_lds_b8 v4, v[2:3], off
+
+// Preserve the original location through macro expansion.
+.macro arrive
+  ds_atomic_async_barrier_arrive_b64 v0
+// WARN: <instantiation>:1:1: warning: no recognized replay workaround
+.endm
+arrive
+s_endpgm
+
+// EOF also diagnoses an outstanding arrival.
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround

>From dda0e7027a7b149707807e1cc7d77ba92e87cce2 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 4 Nov 2026 12:38:13 -0500
Subject: [PATCH 2/4] [AMDGPU] Move async barrier workaround to a separate pass

Run the required pass after scheduling and before wait-count analysis in both pipelines, including O0. Replace the LDS test initializer with poison and update the replay and pipeline tests.
---
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  10 ++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |   2 +
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |   3 +
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 .../SIInsertAsyncBarrierArriveWorkaround.cpp  | 104 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  37 -------
 .../async-barrier-arrive-replay-vgpr-msb.mir  |   8 +-
 .../AMDGPU/async-barrier-arrive-replay.ll     |   5 +-
 .../AMDGPU/async-barrier-arrive-replay.mir    |  10 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll  |   4 +
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   6 +
 11 files changed, 143 insertions(+), 47 deletions(-)
 create mode 100644 llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 75a83c55db2bc9..a575d5e2f53a9b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -107,6 +107,7 @@ FunctionPass *createSIAnnotateControlFlowLegacyPass();
 FunctionPass *createSILowerI1CopiesLegacyPass();
 FunctionPass *createSIShrinkInstructionsLegacyPass();
 FunctionPass *createSIMemoryLegalizerPass();
+FunctionPass *createSIInsertAsyncBarrierArriveWorkaroundPass();
 FunctionPass *createSIInsertWaitcntsPass();
 
 FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
@@ -486,6 +487,13 @@ class AMDGPUMarkLastScratchLoadPass
                         MachineFunctionAnalysisManager &AM);
 };
 
+class SIInsertAsyncBarrierArriveWorkaroundPass
+    : public RequiredPassInfoMixin<SIInsertAsyncBarrierArriveWorkaroundPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
 class SIInsertWaitcntsPass
     : public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
 public:
@@ -599,6 +607,8 @@ extern char &AMDGPULowerVGPREncodingLegacyID;
 void initializeSIInsertHardClausesLegacyPass(PassRegistry &);
 extern char &SIInsertHardClausesID;
 
+void initializeSIInsertAsyncBarrierArriveWorkaroundLegacyPass(PassRegistry &);
+
 void initializeSIInsertWaitcntsLegacyPass(PassRegistry &);
 extern char &SIInsertWaitcntsID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 372d5f5acab21a..177da7b47daf1d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -147,6 +147,8 @@ MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
 MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
 MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
 MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
+MACHINE_FUNCTION_PASS("si-insert-async-barrier-arrive-workaround",
+                      SIInsertAsyncBarrierArriveWorkaroundPass())
 MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
 MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
 MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 9fee82679924e0..96e940efc9aa94 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -735,6 +735,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUInsertDelayAluLegacyPass(*PR);
   initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
   initializeSIInsertHardClausesLegacyPass(*PR);
+  initializeSIInsertAsyncBarrierArriveWorkaroundLegacyPass(*PR);
   initializeSIInsertWaitcntsLegacyPass(*PR);
   initializeSIModeRegisterLegacyPass(*PR);
   initializeSIWholeQuadModeLegacyPass(*PR);
@@ -2033,6 +2034,7 @@ void GCNPassConfig::addPreEmitPass() {
   addPass(createSIMemoryLegalizerPass());
   if (getOptLevel() > CodeGenOptLevel::None)
     addPass(&SIPostRA16BitMovFoldingLegacyID);
+  addPass(createSIInsertAsyncBarrierArriveWorkaroundPass());
   addPass(createSIInsertWaitcntsPass());
 
   addPass(createSIModeRegisterPass());
@@ -2789,6 +2791,7 @@ void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
   addMachineFunctionPass(SIMemoryLegalizerPass(), PMW);
   if (TM.getOptLevel() > CodeGenOptLevel::None)
     addMachineFunctionPass(SIPostRA16BitMovFoldingPass(), PMW);
+  addMachineFunctionPass(SIInsertAsyncBarrierArriveWorkaroundPass(), PMW);
   addMachineFunctionPass(SIInsertWaitcntsPass(), PMW);
 
   addMachineFunctionPass(SIModeRegisterPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index a188c691c29d12..985c8705cf54a6 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -165,6 +165,7 @@ add_llvm_target(AMDGPUCodeGen
   SIFoldOperands.cpp
   SIFormMemoryClauses.cpp
   SIFrameLowering.cpp
+  SIInsertAsyncBarrierArriveWorkaround.cpp
   SIInsertHardClauses.cpp
   SIInsertWaitcnts.cpp
   SIInstrInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
new file mode 100644
index 00000000000000..1ef003e03b85d5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
@@ -0,0 +1,104 @@
+//===- SIInsertAsyncBarrierArriveWorkaround.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert the async barrier arrival replay workaround after scheduling and
+/// before wait-count analysis so the dummy stays immediately after its arrival
+/// and its async counter effects are included in wait generation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-insert-async-barrier-arrive-workaround"
+
+static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  if (!ST.needsAsyncBarrierArriveReplayWorkaround())
+    return false;
+
+  // Collect arrivals before inserting new definitions of ASYNCcnt.
+  SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+    if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+      AsyncBarrierArrivals.push_back(&MI);
+  }
+  if (AsyncBarrierArrivals.empty())
+    return false;
+
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const Register SavedExec = MFI->getSGPRForEXECCopy();
+  if (!SavedExec)
+    report_fatal_error("missing EXEC copy SGPR for async barrier workaround");
+
+  const SIInstrInfo &TII = *ST.getInstrInfo();
+  for (MachineInstr *MI : AsyncBarrierArrivals) {
+    MachineBasicBlock &MBB = *MI->getParent();
+    const DebugLoc &DL = MI->getDebugLoc();
+    auto InsertPt = std::next(MI->getIterator());
+    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+        .addReg(AMDGPU::EXEC_LO);
+    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+        .addImm(0);
+    // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+        .addReg(AMDGPU::VGPR0, RegState::Undef)
+        .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+        .addImm(0)
+        .addImm(0);
+    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+        .addReg(SavedExec);
+  }
+  return true;
+}
+
+namespace {
+class SIInsertAsyncBarrierArriveWorkaroundLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  SIInsertAsyncBarrierArriveWorkaroundLegacy() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    return insertAsyncBarrierArriveWorkaround(MF);
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+} // namespace
+
+INITIALIZE_PASS(SIInsertAsyncBarrierArriveWorkaroundLegacy, DEBUG_TYPE,
+                "SI insert async barrier arrival workaround", false, false)
+
+char SIInsertAsyncBarrierArriveWorkaroundLegacy::ID = 0;
+
+FunctionPass *llvm::createSIInsertAsyncBarrierArriveWorkaroundPass() {
+  return new SIInsertAsyncBarrierArriveWorkaroundLegacy();
+}
+
+PreservedAnalyses SIInsertAsyncBarrierArriveWorkaroundPass::run(
+    MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
+  if (!insertAsyncBarrierArriveWorkaround(MF))
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 3986b87de2a4a6..14067d3e88b7c6 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3498,43 +3498,6 @@ bool SIInsertWaitcnts::run() {
 
   bool Modified = false;
 
-  if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
-    // Collect arrivals before inserting new definitions of ASYNCcnt.
-    SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
-    for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
-      if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
-        AsyncBarrierArrivals.push_back(&MI);
-    }
-
-    if (!AsyncBarrierArrivals.empty()) {
-      const Register SavedExec = MFI->getSGPRForEXECCopy();
-      if (!SavedExec) {
-        report_fatal_error(
-            "missing EXEC copy SGPR for async barrier workaround");
-      }
-
-      for (MachineInstr *MI : AsyncBarrierArrivals) {
-        MachineBasicBlock &MBB = *MI->getParent();
-        const DebugLoc &DL = MI->getDebugLoc();
-        // Insert after scheduling so the dummy stays after its arrival.
-        auto InsertPt = std::next(MI->getIterator());
-        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
-            .addReg(AMDGPU::EXEC_LO);
-        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
-            .addImm(0);
-        // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
-        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
-            .addReg(AMDGPU::VGPR0, RegState::Undef)
-            .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
-            .addImm(0)
-            .addImm(0);
-        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
-            .addReg(SavedExec);
-        Modified = true;
-      }
-    }
-  }
-
   MachineBasicBlock &EntryBB = MF.front();
 
   if (!MFI->isEntryFunction() &&
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
index 155f0d3b296a3a..e496166d213e3a 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
@@ -1,8 +1,8 @@
 # NOTE: Do not autogenerate. Check the bank switch inside the workaround and assemble compiler output with fatal warnings.
-# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
-# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-waitcnts -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
+# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
 
 --- |
   define amdgpu_cs void @high_vgpr_arrive() { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
index 6158c7ec97427e..2ac766ceb848d5 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
@@ -19,8 +19,11 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgpu12.5-amd-amdhsa < %s | FileCheck --check-prefix=AFFECTED %s
 ; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-async-barrier-arrive-replay-bug < %s | FileCheck --check-prefix=UNAFFECTED %s
 
+; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -verify-machineinstrs < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -O0 -enable-new-pm -global-isel=1 -global-isel-abort=1 -mtriple=amdgpu12.50-amd-amdhsa -verify-machineinstrs < %s | FileCheck --check-prefix=AFFECTED %s
+
 ; Check an LDS barrier arrival followed by a global load.
- at bar = internal addrspace(3) global i64 undef, align 8
+ at bar = internal addrspace(3) global i64 poison, align 8
 
 ; AFFECTED-LABEL: replay_window:
 ; AFFECTED: ds_atomic_async_barrier_arrive_b64
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
index 09eb65e1ac5cd8..639f37e3637374 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
@@ -1,9 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
-# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
 
 --- |
   define amdgpu_kernel void @live_scc() { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 36964132edfd75..20c6b0f37005e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -1,3 +1,4 @@
+; NOTE: Do not autogenerate. Check the ordered pass pipeline at each optimization level.
 ; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -O0 -print-pipeline-passes=tree < %s 2>&1 \
 ; RUN:   | FileCheck -check-prefix=GCN-O0 %s
 
@@ -89,6 +90,7 @@
 ; GCN-O0-NEXT:       fentry-insert
 ; GCN-O0-NEXT:       xray-instrumentation
 ; GCN-O0-NEXT:       si-memory-legalizer
+; GCN-O0-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O0-NEXT:       si-insert-waitcnts
 ; GCN-O0-NEXT:       si-mode-register
 ; GCN-O0-NEXT:       si-late-branch-lowering
@@ -274,6 +276,7 @@
 ; GCN-O2-NEXT:       gcn-create-vopd
 ; GCN-O2-NEXT:       si-memory-legalizer
 ; GCN-O2-NEXT:       si-post-ra-16bit-mov-folding
+; GCN-O2-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O2-NEXT:       si-insert-waitcnts
 ; GCN-O2-NEXT:       si-mode-register
 ; GCN-O2-NEXT:       si-insert-hard-clauses
@@ -462,6 +465,7 @@
 ; GCN-O3-NEXT:       gcn-create-vopd
 ; GCN-O3-NEXT:       si-memory-legalizer
 ; GCN-O3-NEXT:       si-post-ra-16bit-mov-folding
+; GCN-O3-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O3-NEXT:       si-insert-waitcnts
 ; GCN-O3-NEXT:       si-mode-register
 ; GCN-O3-NEXT:       si-insert-hard-clauses
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 757f7c9d48a43f..68d08e5004e498 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1,3 +1,4 @@
+; NOTE: Do not autogenerate. Check the ordered pass pipeline at each optimization level.
 ; UNSUPPORTED: expensive_checks
 ; RUN: llc -O0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
 ; RUN:   | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O0 %s
@@ -135,6 +136,7 @@
 ; GCN-O0-NEXT:        Insert fentry calls
 ; GCN-O0-NEXT:        Insert XRay ops
 ; GCN-O0-NEXT:        SI Memory Legalizer
+; GCN-O0-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O0-NEXT:        MachineDominator Tree Construction
 ; GCN-O0-NEXT:        Machine Natural Loop Construction
 ; GCN-O0-NEXT:        MachinePostDominator Tree Construction
@@ -425,6 +427,7 @@
 ; GCN-O1-NEXT:        GCN Create VOPD Instructions
 ; GCN-O1-NEXT:        SI Memory Legalizer
 ; GCN-O1-NEXT:        SI post-RA 16bit Mov Folding
+; GCN-O1-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O1-NEXT:        MachineDominator Tree Construction
 ; GCN-O1-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-NEXT:        MachinePostDominator Tree Construction
@@ -747,6 +750,7 @@
 ; GCN-O1-OPTS-NEXT:        GCN Create VOPD Instructions
 ; GCN-O1-OPTS-NEXT:        SI Memory Legalizer
 ; GCN-O1-OPTS-NEXT:        SI post-RA 16bit Mov Folding
+; GCN-O1-OPTS-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O1-OPTS-NEXT:        MachineDominator Tree Construction
 ; GCN-O1-OPTS-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-OPTS-NEXT:        MachinePostDominator Tree Construction
@@ -1074,6 +1078,7 @@
 ; GCN-O2-NEXT:        GCN Create VOPD Instructions
 ; GCN-O2-NEXT:        SI Memory Legalizer
 ; GCN-O2-NEXT:        SI post-RA 16bit Mov Folding
+; GCN-O2-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O2-NEXT:        MachineDominator Tree Construction
 ; GCN-O2-NEXT:        Machine Natural Loop Construction
 ; GCN-O2-NEXT:        MachinePostDominator Tree Construction
@@ -1416,6 +1421,7 @@
 ; GCN-O3-NEXT:        GCN Create VOPD Instructions
 ; GCN-O3-NEXT:        SI Memory Legalizer
 ; GCN-O3-NEXT:        SI post-RA 16bit Mov Folding
+; GCN-O3-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O3-NEXT:        MachineDominator Tree Construction
 ; GCN-O3-NEXT:        Machine Natural Loop Construction
 ; GCN-O3-NEXT:        MachinePostDominator Tree Construction

>From 7a4ffcfa8110d64c895fb2d10ade9ecd95b0e5ed Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 5 Nov 2026 13:23:44 -0500
Subject: [PATCH 3/4] [AMDGPU] Split async barrier warning into a follow-up

Keep this change focused on the compiler workaround. Remove the assembler
warning and its tests and docs for a separate patch.

Add an example and explain the insertion pass order. Remove the added
include from SIInsertWaitcnts.
---
 .../AMDGPU/async-barrier-replay-warning.c     |  20 --
 .../amdgpu-async-barrier-replay-warning.s     |  15 --
 llvm/docs/AMDGPUUsage.md                      |  28 ---
 llvm/docs/ReleaseNotes.md                     |   7 +-
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  81 --------
 .../SIInsertAsyncBarrierArriveWorkaround.cpp  |  16 +-
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |   1 -
 .../async-barrier-arrive-replay-vgpr-msb.mir  |   4 +-
 .../AMDGPU/async-barrier-replay-inline-asm.ll |  24 ---
 .../async-barrier-replay-warning-vgpr-msb.s   |  38 ----
 .../async-barrier-replay-warning-xnack.s      |  27 ---
 .../MC/AMDGPU/async-barrier-replay-warning.s  | 187 ------------------
 12 files changed, 16 insertions(+), 432 deletions(-)
 delete mode 100644 clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
 delete mode 100644 clang/test/Driver/amdgpu-async-barrier-replay-warning.s
 delete mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
 delete mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
 delete mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
 delete mode 100644 llvm/test/MC/AMDGPU/async-barrier-replay-warning.s

diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
deleted file mode 100644
index b0b57f42c26bd0..00000000000000
--- a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
+++ /dev/null
@@ -1,20 +0,0 @@
-// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
-// REQUIRES: amdgpu-registered-target
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
-
-void unprotected(void) {
-  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
-// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-}
-
-void protected(void) {
-  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
-                   "s_mov_b32 s7, exec_lo\n"
-                   "s_mov_b32 exec_lo, 0\n"
-                   "global_load_async_to_lds_b8 v0, v[0:1], off\n"
-                   "s_mov_b32 exec_lo, s7"
-                   ::: "v0", "v1", "s7", "memory");
-}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
deleted file mode 100644
index 3a1549c3b325e3..00000000000000
--- a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
+++ /dev/null
@@ -1,15 +0,0 @@
-// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
-// REQUIRES: amdgpu-registered-target
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
-// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
-// RUN: cat %s >> %t.s
-// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
-
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index 8a060a0118ae60..218d2dffa302ce 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17222,34 +17222,6 @@ It supports AMDGCN GFX6-GFX11.
 
 This section describes general syntax for instructions and operands.
 
-(amdgpu-async-barrier-replay-warning)=
-
-#### Async Barrier Arrival Warning
-
-The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
-unless XNACK is explicitly disabled. The assembler checks hand-written and
-inline assembly for the same sequence. This check is also omitted when XNACK
-is explicitly disabled.
-
-The assembler warns by default when it cannot recognize the compiler's dummy
-form: `s_mov_b32 exec_lo, 0` followed by
-`global_load_async_to_lds_b8` with a vector address, issued after an arrival
-and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
-not change EXEC, may intervene between the EXEC clear and the load.
-Other load opcodes, scalar-address forms,
-and active loads do not suppress the warning. Recognition stops at labels,
-control-flow instructions, and the end of assembly. Assembler directives
-can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
-block is checked separately. A warning reports missing recognized protection;
-absence of a warning does not prove control-flow safety, correct EXEC
-preservation, or correct async wait counts.
-
-Disable only this check in Clang with
-`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
-`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
-suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
-errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
-
 #### Instructions
 
 An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index e80796200ea92a..658d55971e5687 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,11 +302,8 @@ Makes programs 10x faster by doing Special New Thing.
 
 ### Changes to the AMDGPU Backend
 
-* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
-  assembler warnings for unrecognized sequences in handwritten or inline
-  assembly. Both are omitted when XNACK is explicitly disabled. See
-  {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
-  warning controls.
+* Added a compiler workaround for async barrier arrivals on gfx1250 and gfx1251
+  unless XNACK is explicitly disabled.
 
 * Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
   and `amdgpu.sramecc` module flags.
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 231764b7381153..091279fc95fff2 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,7 +21,6 @@
 #include "Utils/AMDKernelCodeTUtils.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
@@ -41,12 +40,10 @@
 #include "llvm/Support/AMDGPUObjLinkingInfo.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 #include <optional>
-#include <utility>
 
 using namespace llvm;
 using namespace llvm::AMDGPU;
@@ -54,12 +51,6 @@ using namespace llvm::amdhsa;
 
 namespace {
 
-static cl::opt<bool> WarnAsyncBarrierReplay(
-    "amdgpu-asm-warn-async-barrier-replay",
-    cl::desc("Warn about async barrier arrivals without a recognized replay "
-             "workaround in assembly"),
-    cl::init(true), cl::Hidden);
-
 class AMDGPUAsmParser;
 
 enum RegisterKind {
@@ -1144,12 +1135,6 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   const AMDGPU::GPUKind Gfx;
   const AMDGPU::IsaVersion ISA;
 
-  SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
-  bool AsyncBarrierExecCleared = false;
-
-  void checkAsyncBarrierReplay(const MCInst &Inst);
-  void warnPendingAsyncBarrierArrivals();
-
   /// @name Auto-generated Match Functions
   /// {
 
@@ -5854,64 +5839,6 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
   return false;
 }
 
-void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
-  for (SMLoc Loc : PendingAsyncBarrierArrivals) {
-    Warning(Loc, "no recognized replay workaround after "
-                 "'ds_atomic_async_barrier_arrive_b64' on this GPU");
-  }
-  PendingAsyncBarrierArrivals.clear();
-  AsyncBarrierExecCleared = false;
-}
-
-void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
-  if (!WarnAsyncBarrierReplay ||
-      !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
-      getTargetStreamer().getTargetID()->getXnackSetting() ==
-          AMDGPU::TargetIDSetting::Off)
-    return;
-
-  const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
-  switch (Inst.getOpcode()) {
-  case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
-    PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
-    return;
-  // These instructions change control flow but their MC descriptors do not
-  // carry branch, return, or trap flags.
-  case AMDGPU::S_RFE_B64_gfx1250:
-  case AMDGPU::S_TRAP_gfx12:
-    warnPendingAsyncBarrierArrivals();
-    return;
-  case AMDGPU::S_MOV_B32_gfx12:
-    if (!PendingAsyncBarrierArrivals.empty()) {
-      const MCOperand &Dst = Inst.getOperand(0);
-      const MCOperand &Src = Inst.getOperand(1);
-      AsyncBarrierExecCleared = Dst.isReg() &&
-                                mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
-                                Src.isImm() && Src.getImm() == 0;
-    }
-    return;
-  case AMDGPU::S_SET_VGPR_MSB_gfx12:
-    // VGPR encoding can insert this between clearing EXEC and the dummy load.
-    AsyncBarrierExecCleared = ExecCleared;
-    return;
-  case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
-    // Recognize only the compiler's EXEC-zero B8 dummy form.
-    if (ExecCleared)
-      PendingAsyncBarrierArrivals.clear();
-    return;
-  }
-
-  if (PendingAsyncBarrierArrivals.empty())
-    return;
-
-  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
-  // Check only a straight-line sequence. Labels and control-flow changes end
-  // recognition rather than allowing a load on another path to hide a warning.
-  if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
-      (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
-    warnPendingAsyncBarrierArrivals();
-}
-
 bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                               OperandVector &Operands,
                                               MCStreamer &Out,
@@ -5951,7 +5878,6 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       return true;
     }
     emitTargetDirective();
-    checkAsyncBarrierReplay(Inst);
     Out.emitInstruction(Inst, getSTI());
     // Record for kernel prologue checking.
     OpcodeStream.push_back(Inst.getOpcode());
@@ -7096,7 +7022,6 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
 }
 
 void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
-  warnPendingAsyncBarrierArrivals();
   // Record every parsed label in the timeline so that, at end of file, the
   // instructions following a kernel's label can be located regardless of
   // whether the .amdhsa_kernel directive came before or after the label.
@@ -7127,7 +7052,6 @@ void AMDGPUAsmParser::checkKernelPrologues() {
 }
 
 void AMDGPUAsmParser::onEndOfFile() {
-  warnPendingAsyncBarrierArrivals();
   emitTargetDirective();
   checkKernelPrologues();
   if (InfoData)
@@ -7137,11 +7061,6 @@ void AMDGPUAsmParser::onEndOfFile() {
 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getString();
 
-  // Debug locations do not interrupt the instruction sequence. Other
-  // directives may change sections or emit bytes outside this parser.
-  if (IDVal != ".loc" && IDVal != ".file")
-    warnPendingAsyncBarrierArrivals();
-
   if (isHsaAbi(getSTI())) {
     if (IDVal == ".amdhsa_kernel")
       return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
index 1ef003e03b85d5..a03e9e2bf99691 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
@@ -7,9 +7,19 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// Insert the async barrier arrival replay workaround after scheduling and
-/// before wait-count analysis so the dummy stays immediately after its arrival
-/// and its async counter effects are included in wait generation.
+/// Insert the async barrier arrival replay workaround on affected targets
+/// unless XNACK is explicitly disabled. For example, insert the following
+/// sequence after an arrival (s7 illustrates the reserved EXEC-copy SGPR):
+///
+///   ds_atomic_async_barrier_arrive_b64 v2
+///   s_mov_b32 s7, exec_lo
+///   s_mov_b32 exec_lo, 0
+///   global_load_async_to_lds_b8 v0, v[0:1], off
+///   s_mov_b32 exec_lo, s7
+///
+/// Save and restore the active lane mask around the EXEC-zero dummy load.
+/// Run after scheduling to keep the inserted sequence after its arrival,
+/// and before wait-count analysis so it accounts for the added async load.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 14067d3e88b7c6..2db7fc52495448 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,7 +36,6 @@
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/InitializePasses.h"
-#include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 
 using namespace llvm;
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
index e496166d213e3a..521e9f883ecc0e 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
@@ -1,8 +1,6 @@
-# NOTE: Do not autogenerate. Check the bank switch inside the workaround and assemble compiler output with fatal warnings.
+# NOTE: Do not autogenerate. Check the ordered bank switch and EXEC preservation inside the workaround.
 # RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
 # RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | llvm-mc -triple=amdgpu12.50-amd-amdpal -filetype=null --fatal-warnings
-# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | llvm-mc -triple=amdgpu12.51-amd-amdpal -filetype=null --fatal-warnings
 
 --- |
   define amdgpu_cs void @high_vgpr_arrive() { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
deleted file mode 100644
index 0eb8ad252cb4e9..00000000000000
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; NOTE: Do not autogenerate. Object emission invokes the inline assembler; null emission does not. Check warning presence and absence.
-; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -filetype=obj -o %t.o -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
-; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -filetype=obj -o %t.o < %s 2>&1 | count 0
-; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -filetype=obj -o %t.o < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-
-define amdgpu_kernel void @unprotected_inline_asm() {
-  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
-; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-  ret void
-}
-
-define amdgpu_kernel void @protected_inline_asm() {
-  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0\0As_mov_b32 s7, exec_lo\0As_mov_b32 exec_lo, 0\0Aglobal_load_async_to_lds_b8 v0, v[0:1], off\0As_mov_b32 exec_lo, s7", "~{v0},~{v1},~{s7},~{memory}"()
-  ret void
-}
-
-; A later inline-asm block cannot protect an arrival in an earlier block.
-define amdgpu_kernel void @split_inline_asm() {
-  call void asm sideeffect "ds_atomic_async_barrier_arrive_b64 v0", "~{v0},~{memory}"()
-; WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-  call void asm sideeffect "global_load_async_to_lds_b32 v0, v[0:1], off", "~{v0},~{v1},~{memory}"()
-  ret void
-}
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
deleted file mode 100644
index 8a8bff7573ade5..00000000000000
--- a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s
+++ /dev/null
@@ -1,38 +0,0 @@
-// NOTE: Do not autogenerate. Check diagnostic locations and absence across the EXEC-clear sequence.
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
-
-// VGPR bank changes preserve EXEC across the compiler's dummy-load sequence.
-ds_atomic_async_barrier_arrive_b64 v0
-s_mov_b32 s7, exec_lo
-s_mov_b32 exec_lo, 0
-s_set_vgpr_msb 0x100
-global_load_async_to_lds_b8 v0, v[0:1], off
-s_mov_b32 exec_lo, s7
-s_endpgm
-
-// A bank change alone does not establish that EXEC is clear.
-ds_atomic_async_barrier_arrive_b64 v0
-// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_set_vgpr_msb 0x100
-global_load_async_to_lds_b8 v0, v[0:1], off
-s_endpgm
-
-// A bank change does not hide a subsequent write to EXEC.
-ds_atomic_async_barrier_arrive_b64 v0
-// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-s_set_vgpr_msb 0x100
-s_mov_b32 exec_lo, 1
-global_load_async_to_lds_b8 v0, v[0:1], off
-s_endpgm
-
-// Other intervening instructions still break the pattern.
-ds_atomic_async_barrier_arrive_b64 v0
-// CHECK: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-s_nop 0
-s_set_vgpr_msb 0x100
-global_load_async_to_lds_b8 v0, v[0:1], off
-s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
deleted file mode 100644
index f1931aebaef82e..00000000000000
--- a/llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s
+++ /dev/null
@@ -1,27 +0,0 @@
-// NOTE: Do not autogenerate. Check target-ID XNACK settings and warning locations.
-// RUN: split-file %s %t
-// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/on.s 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/off.s 2>&1 | count 0
-// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/on.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | FileCheck %s --check-prefix=ON --implicit-check-not=warning:
-// RUN: sed 's/12.50/12.51/g; s/gfx1250/gfx1251/g' %t/off.s | llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null 2>&1 | count 0
-// RUN: llvm-mc -triple=amdgpu12.50-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.51-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.50s-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.5-amd-amdhsa -filetype=null < %t/any.s 2>&1 | FileCheck %s --check-prefix=ANY --implicit-check-not=warning:
-
-// ON: <stdin>:2:1: warning: no recognized replay workaround
-// ANY: <stdin>:1:1: warning: no recognized replay workaround
-
-//--- on.s
-.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack+"
-ds_atomic_async_barrier_arrive_b64 v0
-s_endpgm
-
-//--- off.s
-.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"
-ds_atomic_async_barrier_arrive_b64 v0
-s_endpgm
-
-//--- any.s
-ds_atomic_async_barrier_arrive_b64 v0
-s_endpgm
diff --git a/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s b/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
deleted file mode 100644
index 719e87544c98a9..00000000000000
--- a/llvm/test/MC/AMDGPU/async-barrier-replay-warning.s
+++ /dev/null
@@ -1,187 +0,0 @@
-// NOTE: Do not autogenerate. Check diagnostic locations and absence across sequence boundaries.
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.51 -filetype=null < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | not llvm-mc -triple=amdgpu12.50 -filetype=null --fatal-warnings 2>&1 | FileCheck %s --check-prefix=FATAL --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=true < %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null -amdgpu-asm-warn-async-barrier-replay=false < %s 2>&1 | count 0
-// RUN: llvm-mc -triple=amdgpu12.50 -filetype=null --no-warn < %s 2>&1 | count 0
-// RUN: echo "ds_atomic_async_barrier_arrive_b64 v0" | llvm-mc -triple=amdgpu13.10 -mattr=+lds-barrier-arrive-atomic -filetype=null 2>&1 | count 0
-// RUN: llvm-mc -triple=amdgpu12.50 -mattr=-async-barrier-arrive-replay-bug -filetype=null < %s 2>&1 | count 0
-// FATAL: <stdin>:1:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-
-// Non-async VMEM ends recognition before the following load.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
-s_nop 0
-global_load_b32 v1, v[2:3], off
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// The compiler's dummy-load sequence is recognized without fixing its SGPR.
-ds_atomic_async_barrier_arrive_b64 v0
-s_mov_b32 s7, exec_lo
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v0, v[0:1], off
-s_mov_b32 exec_lo, s7
-s_endpgm
-
-// Active B8 loads and scalar-address forms are not the recognized dummy form.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-global_load_async_to_lds_b8 v4, v[2:3], off
-global_load_b32 v1, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v5, s[2:3]
-global_load_b32 v1, v[2:3], off
-
-// B32 forms are not recognized, even with EXEC cleared.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b32 v4, v5, s[2:3]
-global_load_b32 v1, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b32 v4, v[2:3], off
-s_endpgm
-
-// Cluster loads do not suppress the warning, even with EXEC cleared.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 m0, 0
-s_mov_b32 exec_lo, 0
-cluster_load_async_to_lds_b8 v4, v[2:3], off
-s_endpgm
-
-// Clearing another register or using a nonzero EXEC is not the dummy form.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 s7, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-s_endpgm
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 1
-global_load_async_to_lds_b8 v4, v[2:3], off
-s_endpgm
-
-// Intervening instructions break the immediately preceding EXEC-clear pattern.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-s_nop 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-s_endpgm
-
-// Source locations do not interrupt a protective sequence.
-.file 1 "async-barrier.s"
-ds_atomic_async_barrier_arrive_b64 v0
-s_mov_b32 s9, exec_lo
-s_mov_b32 exec_lo, 0
-.loc 1 10 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-s_mov_b32 exec_lo, s9
-s_endpgm
-
-// One qualifying load can follow several arrivals.
-ds_atomic_async_barrier_arrive_b64 v0
-ds_atomic_async_barrier_arrive_b64 v1
-s_mov_b32 s9, exec_lo
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-s_mov_b32 exec_lo, s9
-s_endpgm
-
-// B64/B128 forms are not recognized.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b64 v4, v[2:3], off
-global_load_b32 v1, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b128 v4, v[2:3], off
-s_endpgm
-
-// Recognize scratch, flat and buffer VMEM as well as global VMEM.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-scratch_load_b32 v1, v2, off
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-flat_load_b32 v1, v[2:3]
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-buffer_load_b32 v1, v2, s[4:7], s8 offen
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// A load beyond a branch or label does not prove protection on every path.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_branch next
-next:
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_mov_b32 exec_lo, 0
-another_path:
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// Calls, indirect branches, traps and exception returns end recognition.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_swap_pc_i64 s[2:3], s[0:1]
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_set_pc_i64 s[0:1]
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_trap 2
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_rfe_i64 s[0:1]
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// A conditional branch must not let a fall-through load hide the warning.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-s_cbranch_scc0 1
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// Bytes emitted by a directive interrupt recognition.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround
-.long 0
-s_mov_b32 exec_lo, 0
-global_load_async_to_lds_b8 v4, v[2:3], off
-
-// Preserve the original location through macro expansion.
-.macro arrive
-  ds_atomic_async_barrier_arrive_b64 v0
-// WARN: <instantiation>:1:1: warning: no recognized replay workaround
-.endm
-arrive
-s_endpgm
-
-// EOF also diagnoses an outstanding arrival.
-ds_atomic_async_barrier_arrive_b64 v0
-// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround

>From c2a75142df55355add300af6fed7c97f01abc22e Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Sat, 7 Nov 2026 08:44:43 -0500
Subject: [PATCH 4/4] [AMDGPU] Expand async barrier replay during MC emission

Keep the replay as a counted load pseudo until MC emission, where it
expands to the EXEC save, zero-EXEC load, and restore. Preserve register
accounting, instruction size, clause boundaries, bank selection, and
the trailing scalar delay.

Gate the pass by subarchitecture in both pipelines, retain per-function
feature overrides, and simplify EXEC scratch renaming. Replace the small
high-bank MIR-to-assembly test with IR and cover the expansion boundaries.
---
 .../Target/AMDGPU/AMDGPUInsertDelayAlu.cpp    | 13 +++
 llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp  | 83 ++++++++++++-------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 13 ++-
 llvm/lib/Target/AMDGPU/FLATInstructions.td    | 10 +++
 llvm/lib/Target/AMDGPU/SIFrameLowering.cpp    | 10 +--
 .../SIInsertAsyncBarrierArriveWorkaround.cpp  | 35 +++-----
 .../lib/Target/AMDGPU/SIInsertHardClauses.cpp |  3 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  2 +
 .../async-barrier-arrive-replay-clause.mir    | 35 ++++++++
 .../async-barrier-arrive-replay-pipeline.ll   | 21 +++++
 .../async-barrier-arrive-replay-vgpr-msb.ll   | 27 ++++++
 .../async-barrier-arrive-replay-vgpr-msb.mir  | 30 -------
 .../AMDGPU/async-barrier-arrive-replay.ll     | 27 ++++++
 .../AMDGPU/async-barrier-arrive-replay.mir    | 40 +++------
 .../branch-relaxation-inst-size-gfx1250.mir   | 45 ++++++++++
 llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll  |  3 -
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |  5 --
 17 files changed, 274 insertions(+), 128 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-clause.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-pipeline.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.ll
 delete mode 100644 llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index a4083f04768f50..19252f93309274 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -459,6 +459,19 @@ class AMDGPUInsertDelayAlu {
       // twice?
       State.advance(ProducerType, Cycles);
 
+      if (MI.getOpcode() == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay) {
+        // The load clears prior delays, but its MC expansion ends in an EXEC
+        // restore. Model that scalar write without an EXEC def in the MIR.
+        DelayInfo RestoreDelay(
+            SALU, SchedModel->computeInstrLatency(AMDGPU::S_MOV_B32));
+        if (!RestoreDelay.advance(SALU, 1)) {
+          for (MCRegUnit Unit : TRI->regunits(AMDGPU::EXEC_LO))
+            State[Unit] = RestoreDelay;
+        }
+        // A single MIR instruction expands to four ISA instructions.
+        LastDelayAlu = nullptr;
+      }
+
       // Track the preceding WMMA's dst for C-reuse; reset on anything else.
       if (SII->isWMMA(MI) || SII->isSWMMAC(MI)) {
         const MachineOperand *VDst =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index b391761e3b9062..3c0ee0beb0eaa7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -464,37 +464,60 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
             " src1=" + Twine((*V >> 2) & 3) + " src2=" + Twine((*V >> 4) & 3));
     }
 
-    MCInst TmpInst;
-    MCInstLowering.lower(MI, TmpInst);
-    EmitToStreamer(*OutStreamer, TmpInst);
-
-    if (DumpCodeInstEmitter) {
-      // Disassemble instruction/operands to text
-      DisasmLines.resize(DisasmLines.size() + 1);
-      std::string &DisasmLine = DisasmLines.back();
-      raw_string_ostream DisasmStream(DisasmLine);
-
-      AMDGPUInstPrinter InstPrinter(TM.getMCAsmInfo(), *STI.getInstrInfo(),
-                                    *STI.getRegisterInfo());
-      InstPrinter.printInst(&TmpInst, 0, StringRef(), STI, DisasmStream);
-
-      // Disassemble instruction/operands to hex representation.
-      SmallVector<MCFixup, 4> Fixups;
-      SmallVector<char, 16> CodeBytes;
-
-      DumpCodeInstEmitter->encodeInstruction(
-          TmpInst, CodeBytes, Fixups, MF->getSubtarget<MCSubtargetInfo>());
-      HexLines.resize(HexLines.size() + 1);
-      std::string &HexLine = HexLines.back();
-      raw_string_ostream HexStream(HexLine);
-
-      for (size_t i = 0; i < CodeBytes.size(); i += 4) {
-        unsigned int CodeDWord =
-            support::endian::read32le(CodeBytes.data() + i);
-        HexStream << format("%s%08X", (i > 0 ? " " : ""), CodeDWord);
-      }
+    SmallVector<MCInst, 4> Insts;
+    const auto AddMove = [&](MCRegister Dst, MCOperand Src) {
+      MCInst &Move = Insts.emplace_back();
+      Move.setOpcode(STI.getInstrInfo()->pseudoToMCOpcode(AMDGPU::S_MOV_B32));
+      Move.addOperand(MCOperand::createReg(Dst));
+      Move.addOperand(Src);
+    };
+
+    bool IsReplay = Opc == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay;
+    MCRegister SavedExec;
+    if (IsReplay) {
+      SavedExec = AMDGPU::getMCReg(
+          MF->getInfo<SIMachineFunctionInfo>()->getSGPRForEXECCopy(), STI);
+      assert(SavedExec &&
+             "missing EXEC copy SGPR for async barrier workaround");
+      AddMove(SavedExec, MCOperand::createReg(AMDGPU::EXEC_LO));
+      AddMove(AMDGPU::EXEC_LO, MCOperand::createImm(0));
+    }
+
+    MCInstLowering.lower(MI, Insts.emplace_back());
+    if (IsReplay)
+      AddMove(AMDGPU::EXEC_LO, MCOperand::createReg(SavedExec));
+
+    for (const MCInst &Inst : Insts) {
+      EmitToStreamer(*OutStreamer, Inst);
+
+      if (DumpCodeInstEmitter) {
+        // Disassemble instruction/operands to text
+        DisasmLines.resize(DisasmLines.size() + 1);
+        std::string &DisasmLine = DisasmLines.back();
+        raw_string_ostream DisasmStream(DisasmLine);
 
-      DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLine.size());
+        AMDGPUInstPrinter InstPrinter(TM.getMCAsmInfo(), *STI.getInstrInfo(),
+                                      *STI.getRegisterInfo());
+        InstPrinter.printInst(&Inst, 0, StringRef(), STI, DisasmStream);
+
+        // Disassemble instruction/operands to hex representation.
+        SmallVector<MCFixup, 4> Fixups;
+        SmallVector<char, 16> CodeBytes;
+
+        DumpCodeInstEmitter->encodeInstruction(
+            Inst, CodeBytes, Fixups, MF->getSubtarget<MCSubtargetInfo>());
+        HexLines.resize(HexLines.size() + 1);
+        std::string &HexLine = HexLines.back();
+        raw_string_ostream HexStream(HexLine);
+
+        for (size_t i = 0; i < CodeBytes.size(); i += 4) {
+          unsigned int CodeDWord =
+              support::endian::read32le(CodeBytes.data() + i);
+          HexStream << format("%s%08X", (i > 0 ? " " : ""), CodeDWord);
+        }
+
+        DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLine.size());
+      }
     }
   }
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 96e940efc9aa94..4325d9b18ad624 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2028,13 +2028,21 @@ void GCNPassConfig::addPreSched2() {
   addPass(&SIPostRABundlerLegacyID);
 }
 
+static bool mayNeedAsyncBarrierArriveWorkaround(const Triple &TT) {
+  Triple::SubArchType SubArch = TT.getSubArch();
+  return SubArch == Triple::NoSubArch ||
+         SubArch == Triple::AMDGPUSubArch1250S ||
+         AMDGPU::getMajorSubArch(SubArch) == Triple::AMDGPUSubArch12_5;
+}
+
 void GCNPassConfig::addPreEmitPass() {
   if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less))
     addPass(&GCNCreateVOPDID);
   addPass(createSIMemoryLegalizerPass());
   if (getOptLevel() > CodeGenOptLevel::None)
     addPass(&SIPostRA16BitMovFoldingLegacyID);
-  addPass(createSIInsertAsyncBarrierArriveWorkaroundPass());
+  if (mayNeedAsyncBarrierArriveWorkaround(TM->getTargetTriple()))
+    addPass(createSIInsertAsyncBarrierArriveWorkaroundPass());
   addPass(createSIInsertWaitcntsPass());
 
   addPass(createSIModeRegisterPass());
@@ -2791,7 +2799,8 @@ void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
   addMachineFunctionPass(SIMemoryLegalizerPass(), PMW);
   if (TM.getOptLevel() > CodeGenOptLevel::None)
     addMachineFunctionPass(SIPostRA16BitMovFoldingPass(), PMW);
-  addMachineFunctionPass(SIInsertAsyncBarrierArriveWorkaroundPass(), PMW);
+  if (mayNeedAsyncBarrierArriveWorkaround(TM.getTargetTriple()))
+    addMachineFunctionPass(SIInsertAsyncBarrierArriveWorkaroundPass(), PMW);
   addMachineFunctionPass(SIInsertWaitcntsPass(), PMW);
 
   addMachineFunctionPass(SIModeRegisterPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 85427a915c5c63..ee330ba919226d 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1256,6 +1256,16 @@ defm GLOBAL_LOAD_ASYNC_TO_LDS_B64      :  FLAT_Global_Load_LDS_Pseudo<"global_lo
 defm GLOBAL_LOAD_ASYNC_TO_LDS_B128     :  FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b128",  1>;
 } // End SubtargetPredicate = HasAsyncLoadToLDSInsts, WaveSizePredicate = isWave32
 
+// Expand in MC emission into an EXEC save, EXEC-zero load, and EXEC restore.
+// Keep the load's flags and named operands for wait counts and VGPR encoding.
+def GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay :
+    FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b8", 0, 1> {
+  let SubtargetPredicate = HasAsyncLoadToLDSInsts;
+  let WaveSizePredicate = isWave32;
+  let hasSideEffects = 1;
+  let isConvergent = 1;
+}
+
 let SubtargetPredicate = HasAsyncStoreFromLDSInsts in {
 defm GLOBAL_STORE_ASYNC_FROM_LDS_B8    :  FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b8">;
 defm GLOBAL_STORE_ASYNC_FROM_LDS_B32   :  FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b32">;
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index ce761f94b97a2f..a02e63dbc525ae 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1865,13 +1865,11 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
       for (MachineOperand &Op :
            make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
         MachineInstr &MI = *Op.getParent();
-        if (MI.getOpcode() == LMC.OrSaveExecOpc ||
-            (MI.getOpcode() == LMC.MovOpc &&
-             (MI.getOperand(0).getReg() == LMC.ExecReg ||
-              (MI.getOperand(1).isReg() &&
-               MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+        if (MI.getOpcode() != LMC.OrSaveExecOpc && MI.getOpcode() != LMC.MovOpc)
+          continue;
+        if (MI.readsRegister(LMC.ExecReg, TRI) ||
+            MI.modifiesRegister(LMC.ExecReg, TRI))
           Op.setReg(UnusedScratchReg);
-        }
       }
       LiveUnits.addReg(UnusedScratchReg);
     } else {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
index a03e9e2bf99691..d7bd17f19f9870 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertAsyncBarrierArriveWorkaround.cpp
@@ -7,19 +7,10 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// Insert the async barrier arrival replay workaround on affected targets
-/// unless XNACK is explicitly disabled. For example, insert the following
-/// sequence after an arrival (s7 illustrates the reserved EXEC-copy SGPR):
-///
-///   ds_atomic_async_barrier_arrive_b64 v2
-///   s_mov_b32 s7, exec_lo
-///   s_mov_b32 exec_lo, 0
-///   global_load_async_to_lds_b8 v0, v[0:1], off
-///   s_mov_b32 exec_lo, s7
-///
-/// Save and restore the active lane mask around the EXEC-zero dummy load.
-/// Run after scheduling to keep the inserted sequence after its arrival,
-/// and before wait-count analysis so it accounts for the added async load.
+/// Add a counted replay pseudo after affected async barrier arrivals. MC
+/// emission saves EXEC, clears it for the dummy load, and restores it.
+/// Run after scheduling to keep the pseudo after its arrival, and before
+/// wait-count analysis so it accounts for the added async load.
 //
 //===----------------------------------------------------------------------===//
 
@@ -53,26 +44,24 @@ static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) {
 
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   const Register SavedExec = MFI->getSGPRForEXECCopy();
-  if (!SavedExec)
-    report_fatal_error("missing EXEC copy SGPR for async barrier workaround");
+  if (!SavedExec) {
+    reportFatalUsageError(
+        "missing EXEC copy SGPR for async barrier workaround");
+  }
 
   const SIInstrInfo &TII = *ST.getInstrInfo();
   for (MachineInstr *MI : AsyncBarrierArrivals) {
     MachineBasicBlock &MBB = *MI->getParent();
     const DebugLoc &DL = MI->getDebugLoc();
     auto InsertPt = std::next(MI->getIterator());
-    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
-        .addReg(AMDGPU::EXEC_LO);
-    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
-        .addImm(0);
     // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
-    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+    BuildMI(MBB, InsertPt, DL,
+            TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay))
         .addReg(AMDGPU::VGPR0, RegState::Undef)
         .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
         .addImm(0)
-        .addImm(0);
-    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
-        .addReg(SavedExec);
+        .addImm(0)
+        .addReg(SavedExec, RegState::ImplicitDefine);
   }
   return true;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index 80942fd6470ff9..128f4c4192d7f8 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -102,6 +102,9 @@ class SIInsertHardClauses {
   const GCNSubtarget *ST = nullptr;
 
   HardClauseType getHardClauseType(const MachineInstr &MI) {
+    // The replay expands to scalar EXEC moves surrounding its memory operation.
+    if (MI.getOpcode() == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
+      return HARDCLAUSE_ILLEGAL;
     if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) {
       if (ST->hasMixedLoadStoreClauses()) {
         if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 7895c1c8a6a8fe..9186e5d2a1c2bf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -10369,6 +10369,8 @@ unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   unsigned Opc = MI.getOpcode();
   const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
   unsigned DescSize = Desc.getSize();
+  if (Opc == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
+    return DescSize + 12; // Three scalar moves around the load.
 
   // If we have a definitive size, we can use it. Otherwise we need to inspect
   // the operands to know the size.
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-clause.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-clause.mir
new file mode 100644
index 00000000000000..28d9033572b54f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-clause.mir
@@ -0,0 +1,35 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-hard-clauses -o - %s | FileCheck %s --check-prefix=CLAUSE
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-hard-clauses -o - %s | FileCheck %s --check-prefix=CLAUSE
+
+# The replay expands to scalar instructions surrounding a memory operation,
+# so it must remain outside a clause of adjacent real async loads.
+--- |
+  define amdgpu_kernel void @replay_breaks_clause() { ret void }
+...
+
+---
+name: replay_breaks_clause
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr0_vgpr1
+    ; CLAUSE-LABEL: name: replay_breaks_clause
+    ; CLAUSE: liveins: $vgpr0, $vgpr0_vgpr1
+    ; CLAUSE-NEXT: {{  $}}
+    ; CLAUSE-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; CLAUSE-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
+    ; CLAUSE-NEXT: BUNDLE implicit-def $asynccnt, implicit $vgpr0, implicit $vgpr0_vgpr1, implicit $exec, implicit $asynccnt {
+    ; CLAUSE-NEXT:   S_CLAUSE 1
+    ; CLAUSE-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; CLAUSE-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit internal $asynccnt
+    ; CLAUSE-NEXT: }
+    ; CLAUSE-NEXT: S_ENDPGM 0
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    GLOBAL_LOAD_ASYNC_TO_LDS_B8 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    GLOBAL_LOAD_ASYNC_TO_LDS_B8 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-pipeline.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-pipeline.ll
new file mode 100644
index 00000000000000..2d737ec70e3c1e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-pipeline.ll
@@ -0,0 +1,21 @@
+; NOTE: Do not autogenerate. Check subarchitecture gating and ordering before wait insertion.
+; RUN: llc -enable-new-pm -mtriple=amdgpu12.50-amd-amdhsa -O0 -print-pipeline-passes=tree < %s 2>&1 | FileCheck %s --check-prefix=NPM
+; RUN: llc -enable-new-pm -mtriple=amdgpu12.51-amd-amdhsa -print-pipeline-passes=tree < %s 2>&1 | FileCheck %s --check-prefix=NPM
+; RUN: llc -enable-new-pm -mtriple=amdgpu12.50s-amd-amdhsa -print-pipeline-passes=tree < %s 2>&1 | FileCheck %s --check-prefix=NPM
+; RUN: llc -enable-new-pm -mtriple=amdgpu12.5-amd-amdhsa -print-pipeline-passes=tree < %s 2>&1 | FileCheck %s --check-prefix=NPM
+; RUN: llc -enable-new-pm -mtriple=amdgpu13.10-amd-amdhsa -print-pipeline-passes=tree < %s 2>&1 | FileCheck %s --check-prefix=NPM-OFF --implicit-check-not=si-insert-async-barrier-arrive-workaround
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -O0 -debug-pass=Structure < %s 2>&1 | FileCheck %s --check-prefix=LEGACY
+; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -debug-pass=Structure < %s 2>&1 | FileCheck %s --check-prefix=LEGACY
+; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure < %s 2>&1 | FileCheck %s --check-prefix=LEGACY-OFF --implicit-check-not=si-insert-async-barrier-arrive-workaround
+; RUN: llc -mtriple=amdgpu13.10-amd-amdhsa -debug-pass=Structure < %s 2>&1 | FileCheck %s --check-prefix=LEGACY-OFF --implicit-check-not=si-insert-async-barrier-arrive-workaround
+
+; NPM: si-insert-async-barrier-arrive-workaround
+; NPM-NEXT: si-insert-waitcnts
+; NPM-OFF: si-insert-waitcnts
+; LEGACY: SI insert async barrier arrival workaround
+; LEGACY: SI insert wait instructions
+; LEGACY-OFF: SI insert wait instructions
+
+define amdgpu_kernel void @empty() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.ll
new file mode 100644
index 00000000000000..e1d896abf90ea9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.ll
@@ -0,0 +1,27 @@
+; NOTE: Do not autogenerate. Check the ordered VGPR bank switch and contiguous EXEC-preserving expansion.
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s
+
+; Keep a divergent LDS address live across clobbers of all low VGPRs. The
+; occupancy attributes allow allocation of v256 without spilling the address.
+; The arrival uses the high bank, while the replay load needs v[0:1]. Switch
+; banks before the expansion so the EXEC save, load and restore stay together.
+define amdgpu_kernel void @high_vgpr_arrive() #0 {
+; CHECK-LABEL: high_vgpr_arrive:
+; CHECK: s_set_vgpr_msb 0x41
+; CHECK: ds_atomic_async_barrier_arrive_b64 v0 /*v256*/
+; CHECK-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; CHECK-NEXT: s_set_vgpr_msb 0x4100
+; CHECK-NEXT: s_mov_b32 [[SAVED:s[0-9]+]], exec_lo
+; CHECK-NEXT: s_mov_b32 exec_lo, 0
+; CHECK-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
+; CHECK-NEXT: s_mov_b32 exec_lo, [[SAVED]]
+  %lane = call i32 @llvm.amdgcn.workitem.id.x()
+  %address = shl i32 %lane, 3
+  call void asm sideeffect "; clobber low VGPRs", "~{v[0:31]},~{v[32:63]},~{v[64:95]},~{v[96:127]},~{v[128:159]},~{v[160:191]},~{v[192:223]},~{v[224:255]}"()
+  %barrier = inttoptr i32 %address to ptr addrspace(3)
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %barrier)
+  ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,32" "amdgpu-waves-per-eu"="1,1" }
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
deleted file mode 100644
index 521e9f883ecc0e..00000000000000
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# NOTE: Do not autogenerate. Check the ordered bank switch and EXEC preservation inside the workaround.
-# RUN: llc -mtriple=amdgpu12.50-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgpu12.51-amd-amdpal -start-before=si-insert-async-barrier-arrive-workaround -o - %s | FileCheck %s
-
---- |
-  define amdgpu_cs void @high_vgpr_arrive() { ret void }
-...
-
----
-name: high_vgpr_arrive
-tracksRegLiveness: true
-machineFunctionInfo:
-  isEntryFunction: true
-  sgprForEXECCopy: '$sgpr105'
-body: |
-  bb.0:
-    liveins: $vgpr256
-    ; The arrival needs a high VGPR, but the dummy uses v[0:1]. The late
-    ; encoding pass resets the bank after EXEC is cleared.
-    ; CHECK-LABEL: high_vgpr_arrive:
-    ; CHECK: s_set_vgpr_msb 1
-    ; CHECK-NEXT: ds_atomic_async_barrier_arrive_b64 v0 /*v256*/
-    ; CHECK: s_mov_b32 s105, exec_lo
-    ; CHECK-NEXT: s_mov_b32 exec_lo, 0
-    ; CHECK-NEXT: s_set_vgpr_msb 0x100
-    ; CHECK-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off
-    ; CHECK-NEXT: s_mov_b32 exec_lo, s105
-    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr256, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-    S_ENDPGM 0
-...
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
index 2ac766ceb848d5..11fc5a87eefd64 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll
@@ -22,6 +22,17 @@
 ; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -verify-machineinstrs < %s | FileCheck --check-prefix=AFFECTED %s
 ; RUN: llc -O0 -enable-new-pm -global-isel=1 -global-isel-abort=1 -mtriple=amdgpu12.50-amd-amdhsa -verify-machineinstrs < %s | FileCheck --check-prefix=AFFECTED %s
 
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck --check-prefix=AFFECTED %s
+; RUN: llc -enable-new-pm -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1251 < %s | FileCheck --check-prefix=AFFECTED %s
+
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -mattr=+dumpcode -filetype=obj < %s | llvm-objcopy --dump-section .AMDGPU.disasm=- - /dev/null | FileCheck --check-prefix=DUMP %s
+
+; DUMP: ds_atomic_async_barrier_arrive_b64
+; DUMP: s_mov_b32 [[SAVED:s[0-9]+]], exec_lo
+; DUMP-NEXT: s_mov_b32 exec_lo, 0
+; DUMP-NEXT: global_load_async_to_lds_b8 v0, v[0:1], off {{.*}}; {{[0-9A-F]{8} [0-9A-F]{8} ([0-9A-F]{8})}}
+; DUMP-NEXT: s_mov_b32 exec_lo, [[SAVED]]
+
 ; Check an LDS barrier arrival followed by a global load.
 @bar = internal addrspace(3) global i64 poison, align 8
 
@@ -47,3 +58,19 @@ entry:
   store volatile i32 %value, ptr addrspace(1) %dst, align 4
   ret void
 }
+
+; A function can disable the bug feature on an affected module target.
+; AFFECTED-LABEL: function_disabled:
+; AFFECTED: ds_atomic_async_barrier_arrive_b64
+; AFFECTED-NOT: global_load_async_to_lds
+; AFFECTED: s_endpgm
+; UNAFFECTED-LABEL: function_disabled:
+; UNAFFECTED: ds_atomic_async_barrier_arrive_b64
+; UNAFFECTED-NOT: global_load_async_to_lds
+; UNAFFECTED: s_endpgm
+define amdgpu_kernel void @function_disabled() #0 {
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) null)
+  ret void
+}
+
+attributes #0 = { "target-features"="-async-barrier-arrive-replay-bug" }
diff --git a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
index 639f37e3637374..70f9c9f900c426 100644
--- a/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
+++ b/llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir
@@ -1,9 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ON
-# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
-# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=1 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -o - %s | FileCheck %s --check-prefix=ON
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -o - %s | FileCheck %s --check-prefix=OFF
+# RUN: llc -mtriple=amdgpu12.51-amd-amdhsa -amdgpu-xnack=0 -run-pass=si-insert-async-barrier-arrive-workaround,si-insert-waitcnts -o - %s | FileCheck %s --check-prefix=OFF
 
 --- |
   define amdgpu_kernel void @live_scc() { ret void }
@@ -33,10 +33,7 @@ body: |
     ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
     ; ON-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
     ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
-    ; ON-NEXT: $exec_lo = S_MOV_B32 0
-    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
     ; ON-NEXT: $sgpr1 = S_CSELECT_B32 1, 0, implicit $scc
     ; ON-NEXT: S_ENDPGM 0
     ;
@@ -74,10 +71,7 @@ body: |
     ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
     ; ON-NEXT: $exec_lo = S_MOV_B32 0
     ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
-    ; ON-NEXT: $exec_lo = S_MOV_B32 0
-    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
     ; ON-NEXT: $exec_lo = S_MOV_B32 -1
     ; ON-NEXT: S_ENDPGM 0
     ;
@@ -139,20 +133,14 @@ body: |
   ; ON-NEXT:   V_NOP_e32 implicit $exec
   ; ON-NEXT:   GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
   ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
-  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
-  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
   ; ON-NEXT:   S_BRANCH %bb.1
   ; ON-NEXT: {{  $}}
   ; ON-NEXT: bb.1:
   ; ON-NEXT:   liveins: $vgpr0
   ; ON-NEXT: {{  $}}
   ; ON-NEXT:   DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-  ; ON-NEXT:   $sgpr105 = S_MOV_B32 $exec_lo
-  ; ON-NEXT:   $exec_lo = S_MOV_B32 0
-  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-  ; ON-NEXT:   $exec_lo = S_MOV_B32 $sgpr105
+  ; ON-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
   ; ON-NEXT:   S_ENDPGM 0
   ;
   ; OFF-LABEL: name: multiple_arrivals
@@ -201,10 +189,7 @@ body: |
     ; ON-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
     ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
     ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
-    ; ON-NEXT: $exec_lo = S_MOV_B32 0
-    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
     ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
     ; ON-NEXT: S_ENDPGM 0
     ;
@@ -244,10 +229,7 @@ body: |
     ; ON-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
     ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr0, $vgpr0_vgpr1, 0, 0, implicit $exec, implicit-def $asynccnt, implicit $asynccnt
     ; ON-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr0, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
-    ; ON-NEXT: $sgpr105 = S_MOV_B32 $exec_lo
-    ; ON-NEXT: $exec_lo = S_MOV_B32 0
-    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8 undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
-    ; ON-NEXT: $exec_lo = S_MOV_B32 $sgpr105
+    ; ON-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
     ; ON-NEXT: S_WAIT_ASYNCCNT 1, implicit $asynccnt, implicit-def $asynccnt
     ; ON-NEXT: S_ENDPGM 0
     ;
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx1250.mir
index 503119e82b4fb9..29a31cce8f56ea 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx1250.mir
@@ -72,3 +72,48 @@ body: |
   bb.2:
     S_ENDPGM 0
 ...
+
+# The replay includes a 12-byte load and three 4-byte scalar moves. Two
+# replays exceed the 7-dword branch limit, whereas two bare loads fit.
+---
+name: async_barrier_replay_size
+tracksRegLiveness: true
+machineFunctionInfo:
+  sgprForEXECCopy: '$sgpr105'
+body: |
+  ; CHECK-LABEL: name: async_barrier_replay_size
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CMP_EQ_U32 $sgpr8, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64 post-instr-symbol <mcsymbol >
+  ; CHECK-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, target-flags(<unknown target flag>) <mcsymbol >, implicit-def $scc
+  ; CHECK-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, target-flags(<unknown target flag>) <mcsymbol >, implicit-def $scc, implicit $scc
+  ; CHECK-NEXT:   S_SETPC_B64 $sgpr0_sgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
+  ; CHECK-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $sgpr8
+    S_CMP_EQ_U32 $sgpr8, 0, implicit-def $scc
+    S_CBRANCH_SCC0 %bb.2, implicit $scc
+
+  bb.1:
+    GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
+    GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay undef $vgpr0, undef $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt, implicit-def $sgpr105
+
+  bb.2:
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 20c6b0f37005e4..7fcb02dc80b81f 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -90,7 +90,6 @@
 ; GCN-O0-NEXT:       fentry-insert
 ; GCN-O0-NEXT:       xray-instrumentation
 ; GCN-O0-NEXT:       si-memory-legalizer
-; GCN-O0-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O0-NEXT:       si-insert-waitcnts
 ; GCN-O0-NEXT:       si-mode-register
 ; GCN-O0-NEXT:       si-late-branch-lowering
@@ -276,7 +275,6 @@
 ; GCN-O2-NEXT:       gcn-create-vopd
 ; GCN-O2-NEXT:       si-memory-legalizer
 ; GCN-O2-NEXT:       si-post-ra-16bit-mov-folding
-; GCN-O2-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O2-NEXT:       si-insert-waitcnts
 ; GCN-O2-NEXT:       si-mode-register
 ; GCN-O2-NEXT:       si-insert-hard-clauses
@@ -465,7 +463,6 @@
 ; GCN-O3-NEXT:       gcn-create-vopd
 ; GCN-O3-NEXT:       si-memory-legalizer
 ; GCN-O3-NEXT:       si-post-ra-16bit-mov-folding
-; GCN-O3-NEXT:       si-insert-async-barrier-arrive-workaround
 ; GCN-O3-NEXT:       si-insert-waitcnts
 ; GCN-O3-NEXT:       si-mode-register
 ; GCN-O3-NEXT:       si-insert-hard-clauses
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 68d08e5004e498..4e10aa4bd6ef98 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -136,7 +136,6 @@
 ; GCN-O0-NEXT:        Insert fentry calls
 ; GCN-O0-NEXT:        Insert XRay ops
 ; GCN-O0-NEXT:        SI Memory Legalizer
-; GCN-O0-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O0-NEXT:        MachineDominator Tree Construction
 ; GCN-O0-NEXT:        Machine Natural Loop Construction
 ; GCN-O0-NEXT:        MachinePostDominator Tree Construction
@@ -427,7 +426,6 @@
 ; GCN-O1-NEXT:        GCN Create VOPD Instructions
 ; GCN-O1-NEXT:        SI Memory Legalizer
 ; GCN-O1-NEXT:        SI post-RA 16bit Mov Folding
-; GCN-O1-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O1-NEXT:        MachineDominator Tree Construction
 ; GCN-O1-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-NEXT:        MachinePostDominator Tree Construction
@@ -750,7 +748,6 @@
 ; GCN-O1-OPTS-NEXT:        GCN Create VOPD Instructions
 ; GCN-O1-OPTS-NEXT:        SI Memory Legalizer
 ; GCN-O1-OPTS-NEXT:        SI post-RA 16bit Mov Folding
-; GCN-O1-OPTS-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O1-OPTS-NEXT:        MachineDominator Tree Construction
 ; GCN-O1-OPTS-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-OPTS-NEXT:        MachinePostDominator Tree Construction
@@ -1078,7 +1075,6 @@
 ; GCN-O2-NEXT:        GCN Create VOPD Instructions
 ; GCN-O2-NEXT:        SI Memory Legalizer
 ; GCN-O2-NEXT:        SI post-RA 16bit Mov Folding
-; GCN-O2-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O2-NEXT:        MachineDominator Tree Construction
 ; GCN-O2-NEXT:        Machine Natural Loop Construction
 ; GCN-O2-NEXT:        MachinePostDominator Tree Construction
@@ -1421,7 +1417,6 @@
 ; GCN-O3-NEXT:        GCN Create VOPD Instructions
 ; GCN-O3-NEXT:        SI Memory Legalizer
 ; GCN-O3-NEXT:        SI post-RA 16bit Mov Folding
-; GCN-O3-NEXT:        SI insert async barrier arrival workaround
 ; GCN-O3-NEXT:        MachineDominator Tree Construction
 ; GCN-O3-NEXT:        Machine Natural Loop Construction
 ; GCN-O3-NEXT:        MachinePostDominator Tree Construction



More information about the cfe-commits mailing list