[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)

via cfe-commits cfe-commits at lists.llvm.org
Tue Nov 3 13:48:38 PST 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-amdgpu

Author: Yaxun (Sam) Liu (yxsamliu)

<details>
<summary>Changes</summary>

Async barrier arrivals on gfx1250 and gfx1251 currently lack a replay
workaround when XNACK is not explicitly disabled.

Insert the sequence after scheduling, preserving EXEC in a reserved
register. Cover whole-wave functions and warn when handwritten or inline
assembly lacks the recognized sequence.


---

Patch is 76.11 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/238539.diff


20 Files Affected:

- (added) clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c (+20) 
- (added) clang/test/Driver/amdgpu-async-barrier-replay-warning.s (+15) 
- (modified) llvm/docs/AMDGPUUsage.md (+28) 
- (modified) llvm/docs/ReleaseNotes.md (+6) 
- (modified) llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp (+81) 
- (modified) llvm/lib/Target/AMDGPU/GCNFeatures.td (+7) 
- (modified) llvm/lib/Target/AMDGPU/GCNSubtarget.h (+6) 
- (modified) llvm/lib/Target/AMDGPU/SIFrameLowering.cpp (+28-8) 
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+38) 
- (modified) llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp (+8-3) 
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-vgpr-msb.mir (+32) 
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay-whole-wave.ll (+494) 
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.ll (+46) 
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-arrive-replay.mir (+269) 
- (added) llvm/test/CodeGen/AMDGPU/async-barrier-replay-inline-asm.ll (+24) 
- (modified) llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-convergent.ll (+4) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll (+12) 
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning-vgpr-msb.s (+38) 
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning-xnack.s (+27) 
- (added) llvm/test/MC/AMDGPU/async-barrier-replay-warning.s (+187) 


``````````diff
diff --git a/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
new file mode 100644
index 00000000000000..b0b57f42c26bd0
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/async-barrier-replay-warning.c
@@ -0,0 +1,20 @@
+// NOTE: Do not autogenerate. Check raw inline-assembly warnings and Clang suppression in object emission.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -nogpulib -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -mcpu=gfx1250:xnack- -nogpulib -c -o %t.o %s 2>&1 | count 0
+
+void unprotected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0" ::: "v0", "memory");
+// WARN: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+}
+
+void protected(void) {
+  __asm__ volatile("ds_atomic_async_barrier_arrive_b64 v0\n"
+                   "s_mov_b32 s7, exec_lo\n"
+                   "s_mov_b32 exec_lo, 0\n"
+                   "global_load_async_to_lds_b8 v0, v[0:1], off\n"
+                   "s_mov_b32 exec_lo, s7"
+                   ::: "v0", "v1", "s7", "memory");
+}
diff --git a/clang/test/Driver/amdgpu-async-barrier-replay-warning.s b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
new file mode 100644
index 00000000000000..3a1549c3b325e3
--- /dev/null
+++ b/clang/test/Driver/amdgpu-async-barrier-replay-warning.s
@@ -0,0 +1,15 @@
+// NOTE: Do not autogenerate. Check integrated-assembler diagnostics and driver warning controls.
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--no-warn %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=false %s 2>&1 | count 0
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -mllvm -amdgpu-asm-warn-async-barrier-replay=true %s 2>&1 | FileCheck %s --check-prefix=WARN --implicit-check-not=warning:
+// RUN: not %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o -Wa,--fatal-warnings %s 2>&1 | FileCheck %s --check-prefix=ERROR --implicit-check-not=warning:
+// RUN: echo '.amdgcn_target "amdgpu12.50-amd-amdhsa-unknown-gfx1250:xnack-"' > %t.s
+// RUN: cat %s >> %t.s
+// RUN: %clang --target=amdgpu12.50-amd-amdhsa -c -o %t.o %t.s 2>&1 | count 0
+
+ds_atomic_async_barrier_arrive_b64 v0
+// WARN: :[[@LINE-1]]:1: warning: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+// ERROR: :[[@LINE-2]]:1: error: no recognized replay workaround after 'ds_atomic_async_barrier_arrive_b64' on this GPU
+s_endpgm
diff --git a/llvm/docs/AMDGPUUsage.md b/llvm/docs/AMDGPUUsage.md
index fe5832b286a2b3..86ed4db05ce05f 100644
--- a/llvm/docs/AMDGPUUsage.md
+++ b/llvm/docs/AMDGPUUsage.md
@@ -17109,6 +17109,34 @@ It supports AMDGCN GFX6-GFX11.
 
 This section describes general syntax for instructions and operands.
 
+(amdgpu-async-barrier-replay-warning)=
+
+#### Async Barrier Arrival Warning
+
+The compiler emits an async barrier arrival workaround for gfx1250 and gfx1251
+unless XNACK is explicitly disabled. The assembler checks hand-written and
+inline assembly for the same sequence. This check is also omitted when XNACK
+is explicitly disabled.
+
+The assembler warns by default when it cannot recognize the compiler's dummy
+form: `s_mov_b32 exec_lo, 0` followed by
+`global_load_async_to_lds_b8` with a vector address, issued after an arrival
+and before a non-async VMEM instruction. Only `s_set_vgpr_msb`, which does
+not change EXEC, may intervene between the EXEC clear and the load.
+Other load opcodes, scalar-address forms,
+and active loads do not suppress the warning. Recognition stops at labels,
+control-flow instructions, and the end of assembly. Assembler directives
+can also end recognition; `.loc` and `.file` preserve it. Each inline assembly
+block is checked separately. A warning reports missing recognized protection;
+absence of a warning does not prove control-flow safety, correct EXEC
+preservation, or correct async wait counts.
+
+Disable only this check in Clang with
+`-mllvm -amdgpu-asm-warn-async-barrier-replay=false`, or in `llvm-mc` with
+`-amdgpu-asm-warn-async-barrier-replay=false`. Clang's `-Wa,--no-warn`
+suppresses all assembler warnings, and `-Wa,--fatal-warnings` treats them as
+errors. With `llvm-mc`, use `--no-warn` and `--fatal-warnings` respectively.
+
 #### Instructions
 
 An instruction has the following [syntax](AMDGPUInstructionSyntax.md):
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 75318319d88b92..bbf4618825417f 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,12 @@ Makes programs 10x faster by doing Special New Thing.
 
 ### Changes to the AMDGPU Backend
 
+* Added an async barrier arrival workaround for gfx1250 and gfx1251, and
+  assembler warnings for unrecognized sequences in handwritten or inline
+  assembly. Both are omitted when XNACK is explicitly disabled. See
+  {ref}`amdgpu-async-barrier-replay-warning` for recognized sequences and
+  warning controls.
+
 * Replaced `xnack` and `sramecc` target features with `amdgpu.xnack`
   and `amdgpu.sramecc` module flags.
 * `llvm.amdgcn.make.buffer.rsrc` now accepts any integer width for its
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index a8c6d5ef457639..19314b0f240be3 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
 #include "Utils/AMDKernelCodeTUtils.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
@@ -40,10 +41,12 @@
 #include "llvm/Support/AMDGPUObjLinkingInfo.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 #include <optional>
+#include <utility>
 
 using namespace llvm;
 using namespace llvm::AMDGPU;
@@ -51,6 +54,12 @@ using namespace llvm::amdhsa;
 
 namespace {
 
+static cl::opt<bool> WarnAsyncBarrierReplay(
+    "amdgpu-asm-warn-async-barrier-replay",
+    cl::desc("Warn about async barrier arrivals without a recognized replay "
+             "workaround in assembly"),
+    cl::init(true), cl::Hidden);
+
 class AMDGPUAsmParser;
 
 enum RegisterKind {
@@ -1135,6 +1144,12 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   const AMDGPU::GPUKind Gfx;
   const AMDGPU::IsaVersion ISA;
 
+  SmallVector<SMLoc, 4> PendingAsyncBarrierArrivals;
+  bool AsyncBarrierExecCleared = false;
+
+  void checkAsyncBarrierReplay(const MCInst &Inst);
+  void warnPendingAsyncBarrierArrivals();
+
   /// @name Auto-generated Match Functions
   /// {
 
@@ -5839,6 +5854,64 @@ static bool isInvalidVOPDY(const OperandVector &Operands,
   return false;
 }
 
+void AMDGPUAsmParser::warnPendingAsyncBarrierArrivals() {
+  for (SMLoc Loc : PendingAsyncBarrierArrivals) {
+    Warning(Loc, "no recognized replay workaround after "
+                 "'ds_atomic_async_barrier_arrive_b64' on this GPU");
+  }
+  PendingAsyncBarrierArrivals.clear();
+  AsyncBarrierExecCleared = false;
+}
+
+void AMDGPUAsmParser::checkAsyncBarrierReplay(const MCInst &Inst) {
+  if (!WarnAsyncBarrierReplay ||
+      !getSTI().hasFeature(AMDGPU::FeatureAsyncBarrierArriveReplayBug) ||
+      getTargetStreamer().getTargetID()->getXnackSetting() ==
+          AMDGPU::TargetIDSetting::Off)
+    return;
+
+  const bool ExecCleared = std::exchange(AsyncBarrierExecCleared, false);
+  switch (Inst.getOpcode()) {
+  case AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64_gfx12:
+    PendingAsyncBarrierArrivals.push_back(Inst.getLoc());
+    return;
+  // These instructions change control flow but their MC descriptors do not
+  // carry branch, return, or trap flags.
+  case AMDGPU::S_RFE_B64_gfx1250:
+  case AMDGPU::S_TRAP_gfx12:
+    warnPendingAsyncBarrierArrivals();
+    return;
+  case AMDGPU::S_MOV_B32_gfx12:
+    if (!PendingAsyncBarrierArrivals.empty()) {
+      const MCOperand &Dst = Inst.getOperand(0);
+      const MCOperand &Src = Inst.getOperand(1);
+      AsyncBarrierExecCleared = Dst.isReg() &&
+                                mc2PseudoReg(Dst.getReg()) == AMDGPU::EXEC_LO &&
+                                Src.isImm() && Src.getImm() == 0;
+    }
+    return;
+  case AMDGPU::S_SET_VGPR_MSB_gfx12:
+    // VGPR encoding can insert this between clearing EXEC and the dummy load.
+    AsyncBarrierExecCleared = ExecCleared;
+    return;
+  case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_gfx1250:
+    // Recognize only the compiler's EXEC-zero B8 dummy form.
+    if (ExecCleared)
+      PendingAsyncBarrierArrivals.clear();
+    return;
+  }
+
+  if (PendingAsyncBarrierArrivals.empty())
+    return;
+
+  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
+  // Check only a straight-line sequence. Labels and control-flow changes end
+  // recognition rather than allowing a load on another path to hide a warning.
+  if (Desc.isBranch() || Desc.isCall() || Desc.isTerminator() ||
+      (SIInstrFlags::isVMEM(Desc) && !SIInstrFlags::usesASYNC_CNT(Desc)))
+    warnPendingAsyncBarrierArrivals();
+}
+
 bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                               OperandVector &Operands,
                                               MCStreamer &Out,
@@ -5878,6 +5951,7 @@ bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       return true;
     }
     emitTargetDirective();
+    checkAsyncBarrierReplay(Inst);
     Out.emitInstruction(Inst, getSTI());
     // Record for kernel prologue checking.
     OpcodeStream.push_back(Inst.getOpcode());
@@ -7027,6 +7101,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
 }
 
 void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
+  warnPendingAsyncBarrierArrivals();
   // Record every parsed label in the timeline so that, at end of file, the
   // instructions following a kernel's label can be located regardless of
   // whether the .amdhsa_kernel directive came before or after the label.
@@ -7057,6 +7132,7 @@ void AMDGPUAsmParser::checkKernelPrologues() {
 }
 
 void AMDGPUAsmParser::onEndOfFile() {
+  warnPendingAsyncBarrierArrivals();
   emitTargetDirective();
   checkKernelPrologues();
   if (InfoData)
@@ -7066,6 +7142,11 @@ void AMDGPUAsmParser::onEndOfFile() {
 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getString();
 
+  // Debug locations do not interrupt the instruction sequence. Other
+  // directives may change sections or emit bytes outside this parser.
+  if (IDVal != ".loc" && IDVal != ".file")
+    warnPendingAsyncBarrierArrivals();
+
   if (isHsaAbi(getSTI())) {
     if (IDVal == ".amdhsa_kernel")
       return ParseDirectiveAMDHSAKernel();
diff --git a/llvm/lib/Target/AMDGPU/GCNFeatures.td b/llvm/lib/Target/AMDGPU/GCNFeatures.td
index 313f67ea31e22f..cc8ed4cdd251f3 100644
--- a/llvm/lib/Target/AMDGPU/GCNFeatures.td
+++ b/llvm/lib/Target/AMDGPU/GCNFeatures.td
@@ -265,6 +265,12 @@ def FeaturePreciseMemory
     : SubtargetFeature<"precise-memory", "EnablePreciseMemory",
                        "true", "Enable precise memory mode">;
 
+defm AsyncBarrierArriveReplayBug : AMDGPUSubtargetFeature<
+  "async-barrier-arrive-replay-bug",
+  "Enable the async barrier arrival workaround",
+  /*GenPredicate=*/0, /*GenAssemblerPredicate=*/0, [], InlineIgnore
+>;
+
 defm SGPRInitBug : AMDGPUSubtargetFeature<"sgpr-init-bug",
   "VI SGPR initialization bug requiring a fixed SGPR allocation size",
   /*GenPredicate=*/1, /*GenAssemblerPredicate=*/1, [], InlineIgnore
@@ -2493,6 +2499,7 @@ def FeatureISAVersion12 : FeatureSet<
 
 def FeatureISAVersion12_50_Common : FeatureSet<
   [FeatureGFX12,
+   FeatureAsyncBarrierArriveReplayBug,
    FeatureGFX1250Insts,
    FeatureBackOffBarrier,
    FeatureRequiresAlignedVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 38604fb4741b36..07829fcc1486cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -345,6 +345,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool isXNACKEnabled() const { return TargetID.isXnackOnOrAny(); }
 
+  bool needsAsyncBarrierArriveReplayWorkaround() const {
+    // Generic 12.5 code has no explicit XNACK setting; keep the workaround.
+    return hasAsyncBarrierArriveReplayBug() &&
+           TargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Off;
+  }
+
   bool hasRelaxedBufferOOBMode() const { return BufferOOBRelaxed; }
   bool hasRelaxedTBufferOOBMode() const { return TBufferOOBRelaxed; }
 
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index cb51eeb8b13c96..ce761f94b97a2f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1859,7 +1859,20 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
       // If found any unused scratch SGPR, reserve the register itself for Exec
       // copy and there is no need for any spills in that case.
       MFI->setSGPRForEXECCopy(UnusedScratchReg);
-      MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
+      // Only rewrite EXEC saves and restores. Other uses, such as inline
+      // assembly operands and their callee saves, name fixed registers.
+      const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+      for (MachineOperand &Op :
+           make_early_inc_range(MRI.reg_operands(ReservedRegForExecCopy))) {
+        MachineInstr &MI = *Op.getParent();
+        if (MI.getOpcode() == LMC.OrSaveExecOpc ||
+            (MI.getOpcode() == LMC.MovOpc &&
+             (MI.getOperand(0).getReg() == LMC.ExecReg ||
+              (MI.getOperand(1).isReg() &&
+               MI.getOperand(1).getReg() == LMC.ExecReg)))) {
+          Op.setReg(UnusedScratchReg);
+        }
+      }
       LiveUnits.addReg(UnusedScratchReg);
     } else {
       // Needs spill.
@@ -1933,19 +1946,23 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
   const SIInstrInfo *TII = ST.getInstrInfo();
   bool NeedExecCopyReservedReg = false;
+  bool NeedExecCopyForAsyncBarrier = false;
 
   MachineInstr *ReturnMI = nullptr;
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
       // TODO: Walking through all MBBs here would be a bad heuristic. Better
       // handle them elsewhere.
-      if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+      if (TII->isWWMRegSpillOpcode(MI.getOpcode())) {
         NeedExecCopyReservedReg = true;
-      else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
-               MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
-               MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
-               (MFI->isChainFunction() &&
-                TII->isChainCallOpcode(MI.getOpcode()))) {
+      } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
+                 ST.needsAsyncBarrierArriveReplayWorkaround()) {
+        NeedExecCopyForAsyncBarrier = true;
+      } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
+                 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
+                 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
+                 (MFI->isChainFunction() &&
+                  TII->isChainCallOpcode(MI.getOpcode()))) {
         // We expect all return to be the same size.
         assert(!ReturnMI ||
                (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
@@ -2013,7 +2030,10 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
   if (!ST.hasGFX90AInsts())
     SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
 
-  determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
+  // Whole-wave spills use the original EXEC mask from the first argument, but
+  // the barrier workaround needs separate storage for the current EXEC mask.
+  determinePrologEpilogSGPRSaves(
+      MF, SavedVGPRs, NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
 
   // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
   // allow the default insertion to handle them.
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bd2815d0491b5b..9fa338a3f1826f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/AMDGPUTargetParser.h"
 
 using namespace llvm;
@@ -3497,6 +3498,43 @@ bool SIInsertWaitcnts::run() {
 
   bool Modified = false;
 
+  if (ST.needsAsyncBarrierArriveReplayWorkaround()) {
+    // Collect arrivals before inserting new definitions of ASYNCcnt.
+    SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+    for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+      if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+        AsyncBarrierArrivals.push_back(&MI);
+    }
+
+    if (!AsyncBarrierArrivals.empty()) {
+      const Register SavedExec = MFI->getSGPRForEXECCopy();
+      if (!SavedExec) {
+        report_fatal_error(
+            "missing EXEC copy SGPR for async barrier workaround");
+      }
+
+      for (MachineInstr *MI : AsyncBarrierArrivals) {
+        MachineBasicBlock &MBB = *MI->getParent();
+        const DebugLoc &DL = MI->getDebugLoc();
+        // Insert after scheduling so the dummy stays after its arrival.
+        auto InsertPt = std::next(MI->getIterator());
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), SavedExec)
+            .addReg(AMDGPU::EXEC_LO);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addImm(0);
+        // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8))
+            .addReg(AMDGPU::VGPR0, RegState::Undef)
+            .addReg(AMDGPU::VGPR0_VGPR1, RegState::Undef)
+            .addImm(0)
+            .addImm(0);
+        BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::EXEC_LO)
+            .addReg(SavedExec);
+        Modified = true;
+      }
+    }
+  }
+
   MachineBasicBlock &EntryBB = MF.front();
 
   if (!MFI->isEntryFunction() &&
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index d0a57e163d29a0..f48e2194242a42 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -619,9 +619,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
     if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
                              TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
       FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
-  } else {
-    // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
-    // spills/copies. Reset the SGPR reserved for EXEC copy.
+  } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
+             none_of(MRI.def_instructions(AMDGPU::ASYNCcnt),
+                     [](const MachineInstr &MI) {
+                       return MI.getOpcode() ==
+                              AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
+                     })) {
+    // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
+    // needs the reserved register for EXEC copy.
    ...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/238539


More information about the cfe-commits mailing list