[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)

Yaxun Liu via cfe-commits cfe-commits at lists.llvm.org
Sat Nov 7 05:54:23 PST 2026


================
@@ -0,0 +1,114 @@
+//===- SIInsertAsyncBarrierArriveWorkaround.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert the async barrier arrival replay workaround on affected targets
+/// unless XNACK is explicitly disabled. For example, insert the following
+/// sequence after an arrival (s7 illustrates the reserved EXEC-copy SGPR):
+///
+///   ds_atomic_async_barrier_arrive_b64 v2
+///   s_mov_b32 s7, exec_lo
+///   s_mov_b32 exec_lo, 0
+///   global_load_async_to_lds_b8 v0, v[0:1], off
+///   s_mov_b32 exec_lo, s7
+///
+/// Save and restore the active lane mask around the EXEC-zero dummy load.
+/// Run after scheduling to keep the inserted sequence after its arrival,
+/// and before wait-count analysis so it accounts for the added async load.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-insert-async-barrier-arrive-workaround"
+
+static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  if (!ST.needsAsyncBarrierArriveReplayWorkaround())
+    return false;
+
+  // Collect arrivals before inserting new definitions of ASYNCcnt.
+  SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+    if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+      AsyncBarrierArrivals.push_back(&MI);
+  }
+  if (AsyncBarrierArrivals.empty())
+    return false;
+
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const Register SavedExec = MFI->getSGPRForEXECCopy();
+  if (!SavedExec)
+    report_fatal_error("missing EXEC copy SGPR for async barrier workaround");
----------------
yxsamliu wrote:

Changed to `reportFatalUsageError`.

https://github.com/llvm/llvm-project/pull/238539


More information about the cfe-commits mailing list