[clang] [llvm] [AMDGPU] Add an async barrier arrival workaround (PR #238539)
Yaxun Liu via cfe-commits
cfe-commits at lists.llvm.org
Sat Nov 7 05:54:23 PST 2026
================
@@ -0,0 +1,114 @@
+//===- SIInsertAsyncBarrierArriveWorkaround.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert the async barrier arrival replay workaround on affected targets
+/// unless XNACK is explicitly disabled. For example, insert the following
+/// sequence after an arrival (s7 illustrates the reserved EXEC-copy SGPR):
+///
+/// ds_atomic_async_barrier_arrive_b64 v2
+/// s_mov_b32 s7, exec_lo
+/// s_mov_b32 exec_lo, 0
+/// global_load_async_to_lds_b8 v0, v[0:1], off
+/// s_mov_b32 exec_lo, s7
+///
+/// Save and restore the active lane mask around the EXEC-zero dummy load.
+/// Run after scheduling to keep the inserted sequence after its arrival,
+/// and before wait-count analysis so it accounts for the added async load.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-insert-async-barrier-arrive-workaround"
+
+static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.needsAsyncBarrierArriveReplayWorkaround())
+ return false;
+
+ // Collect arrivals before inserting new definitions of ASYNCcnt.
+ SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+ for (MachineInstr &MI : MRI.def_instructions(AMDGPU::ASYNCcnt)) {
+ if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+ AsyncBarrierArrivals.push_back(&MI);
+ }
+ if (AsyncBarrierArrivals.empty())
+ return false;
+
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ const Register SavedExec = MFI->getSGPRForEXECCopy();
+ if (!SavedExec)
+ report_fatal_error("missing EXEC copy SGPR for async barrier workaround");
----------------
yxsamliu wrote:
Changed to `reportFatalUsageError`.
https://github.com/llvm/llvm-project/pull/238539
More information about the cfe-commits
mailing list