[Mlir-commits] [clang] [llvm] [mlir] [AMDGPU] Implement AsyncMark Stages (PR #220442)
Ryan Mitchell
llvmlistbot at llvm.org
Thu Oct 29 08:57:24 PDT 2026
https://github.com/RyanRio updated https://github.com/llvm/llvm-project/pull/220442
>From b77815da355f91bb9c402fdfea166a6872f271a5 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Tue, 1 Sep 2026 16:13:10 -0700
Subject: [PATCH 1/3] Enables asyncmark operations to use bitmasks.
Append and remove asyncmarks from per-stage sequences, instead of a
single sequence. Both llvm.amdgcn.asyncmark and llvm.amdgcn.wait.asyncmark
take a constant stage mask in which a set bit names a stage; the operation
acts only on the sequences of the stages it names.
The zero mask is special and names every stage, which preserves the
original behavior; old IR is auto-upgraded to pass a zero mask.
Each async operation belongs to exactly one stage, determined by the
instruction that initiates it. The hardware counters waited on are
implied by the operations in each stage, and stages are split so that
waiting for a mark in one stage never requires waiting on a mark in another.
---
clang/include/clang/Basic/BuiltinsAMDGPU.td | 12 +-
clang/lib/Sema/SemaAMDGPU.cpp | 17 +
.../builtins-amdgcn-asyncmark-errs-gfx1250.cl | 30 +
.../builtins-amdgcn-asyncmark-errs.cl | 4 +-
.../builtins-amdgcn-asyncmark.cl | 37 +-
llvm/docs/AMDGPUAsyncOperations.md | 257 ++++--
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 17 +-
llvm/include/llvm/Support/AMDGPUAsyncStages.h | 205 +++++
llvm/lib/IR/AutoUpgrade.cpp | 29 +
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 19 +-
llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp | 101 +++
llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h | 6 +
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 293 ++++---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 53 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 8 +
llvm/lib/Target/AMDGPU/SOPInstructions.td | 8 +-
llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll | 58 ++
.../test/CodeGen/AMDGPU/async-buffer-loads.ll | 52 +-
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 200 ++---
.../CodeGen/AMDGPU/asyncmark-loop-carried.mir | 227 +++++
.../CodeGen/AMDGPU/asyncmark-max-pregfx12.ll | 167 ++--
.../AMDGPU/asyncmark-merge-empty-other.mir | 28 +-
.../asyncmark-merge-rebase-pregfx12.mir | 16 +-
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 200 ++---
.../CodeGen/AMDGPU/asyncmark-stage-err.ll | 30 +
.../CodeGen/AMDGPU/asyncmark-stage-max.mir | 219 +++++
.../CodeGen/AMDGPU/asyncmark-stage-merge.mir | 285 +++++++
.../AMDGPU/asyncmark-stage-pregfx12.ll | 115 +++
llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll | 784 ++++++++++++++++++
.../test/CodeGen/AMDGPU/asyncmark-waitcnt.mir | 4 +-
.../CodeGen/AMDGPU/code-size-estimate.mir | 4 +-
.../AMDGPU/llvm.amdgcn.tensor.load.store.ll | 78 +-
.../AMDGPU/asyncmark-stage-err.mir | 29 +
.../AMDGPU/tensor-descriptor.ll | 16 +-
.../GVN/AMDGPU/tensor-descriptor.ll | 16 +-
.../include/mlir/Dialect/LLVMIR/ROCDLAttrs.td | 3 +
.../include/mlir/Dialect/LLVMIR/ROCDLEnums.td | 25 +
mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 28 +-
mlir/test/Dialect/LLVMIR/invalid.mlir | 7 +
mlir/test/Dialect/LLVMIR/rocdl.mlir | 11 +-
mlir/test/Target/LLVMIR/rocdl.mlir | 8 +-
41 files changed, 3095 insertions(+), 611 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
create mode 100644 llvm/include/llvm/Support/AMDGPUAsyncStages.h
create mode 100644 llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-loop-carried.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
create mode 100644 llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 5a0b71fa50c19b..8853f977118bbb 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -334,8 +334,16 @@ def __builtin_amdgcn_av_store_b128
//===----------------------------------------------------------------------===//
// Mirrors GCNSubtarget::hasAsyncMark()
-def __builtin_amdgcn_asyncmark : AMDGPUBuiltin<"void()", [], "vmem-to-lds-load-insts|asynccnt">;
-def __builtin_amdgcn_wait_asyncmark : AMDGPUBuiltin<"void(_Constant unsigned short)", [], "vmem-to-lds-load-insts|asynccnt">;
+def __builtin_amdgcn_asyncmark
+ : AMDGPUBuiltin<"void(_Constant unsigned int)", [],
+ "vmem-to-lds-load-insts|asynccnt"> {
+ let ArgNames = ["stage_mask"];
+}
+def __builtin_amdgcn_wait_asyncmark
+ : AMDGPUBuiltin<"void(_Constant unsigned short, _Constant unsigned int)", [],
+ "vmem-to-lds-load-insts|asynccnt"> {
+ let ArgNames = ["num_marks", "stage_mask"];
+}
//===----------------------------------------------------------------------===//
// Ballot builtins.
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index b1f6fa33649bd1..573dc662a9fbfa 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -25,6 +25,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
#include "llvm/TargetParser/AtomicScope.h"
@@ -220,6 +221,22 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(const TargetInfo &TI,
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_bf6:
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_fp6:
return SemaRef.BuiltinConstantArgRange(TheCall, 2, 0, 15);
+ case AMDGPU::BI__builtin_amdgcn_asyncmark:
+ case AMDGPU::BI__builtin_amdgcn_wait_asyncmark: {
+ bool IsMark = BuiltinID == AMDGPU::BI__builtin_amdgcn_asyncmark;
+
+ // Check the sequence length limit is a constant.
+ llvm::APSInt NumMarks;
+ if (!IsMark && SemaRef.BuiltinConstantArg(TheCall, 0, NumMarks))
+ return true;
+
+ // The stage mask names the stages to act on, so every combination of known
+ // stage bits is meaningful, including none of them: the zero mask names
+ // every stage.
+ unsigned MaskArgNum = IsMark ? 0 : 1;
+ return SemaRef.BuiltinConstantArgRange(TheCall, MaskArgNum, 0,
+ llvm::AMDGPU::AsyncStages::ALL);
+ }
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
case AMDGPU::BI__builtin_amdgcn_av_store_b128:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
new file mode 100644
index 00000000000000..1946da31d4861f
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
@@ -0,0 +1,30 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.50-amd-amdhsa -verify -S -o - %s
+
+// The stage mask must be a compile-time constant.
+
+void test_mask_not_constant(unsigned int m) {
+ __builtin_amdgcn_asyncmark(m); // expected-error{{argument to '__builtin_amdgcn_asyncmark' must be a constant integer}}
+ __builtin_amdgcn_wait_asyncmark(0, m); // expected-error{{argument to '__builtin_amdgcn_wait_asyncmark' must be a constant integer}}
+}
+
+// A mask may only name stages that exist. 2047 names all eleven of them, and
+// 2048 is the bit just past the last one.
+
+void test_mask_out_of_range() {
+ __builtin_amdgcn_asyncmark(2048); // expected-error{{argument value 2048 is outside the valid range [0, 2047]}}
+ __builtin_amdgcn_asyncmark(4096); // expected-error{{argument value 4096 is outside the valid range [0, 2047]}}
+ __builtin_amdgcn_wait_asyncmark(0, 2048); // expected-error{{argument value 2048 is outside the valid range [0, 2047]}}
+}
+
+// Every combination of in-range bits is accepted, including bits of the
+// reserved stages: naming a stage whose operations do not exist yet is
+// harmless, and keeps masks portable as the reserved slots are filled in.
+
+void test_mask_reserved_bits_ok() {
+ __builtin_amdgcn_asyncmark(16); // RESERVED_4
+ __builtin_amdgcn_asyncmark(2000); // every reserved stage at once
+ __builtin_amdgcn_asyncmark(2047); // every stage
+ __builtin_amdgcn_wait_asyncmark(0, 16);
+ __builtin_amdgcn_wait_asyncmark(0, 2047);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
index 63d68a7c30433f..55f967b9bdb6e3 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
@@ -3,6 +3,6 @@
// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.00-amd-amdhsa -verify -S -o - %s
void test_feature() {
- __builtin_amdgcn_asyncmark(); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
- __builtin_amdgcn_wait_asyncmark(0); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_asyncmark(0); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_wait_asyncmark(0, 0); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
index 838bfdbdcb8f15..c5bb8aadbd398d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
@@ -5,13 +5,42 @@
// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -triple amdgpu12.50-unknown-unknown -emit-llvm -o - %s | FileCheck %s
// REQUIRES: amdgpu-registered-target
+// The argument is a mask of the stages to act on, and reaches the intrinsic
+// unchanged. A zero mask is special and covers every stage.
+
// CHECK-LABEL: @test_invocation(
// CHECK-NEXT: entry:
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
// CHECK-NEXT: ret void
//
void test_invocation() {
- __builtin_amdgcn_asyncmark();
- __builtin_amdgcn_wait_asyncmark(0);
+ __builtin_amdgcn_asyncmark(0);
+ __builtin_amdgcn_wait_asyncmark(0, 0);
+}
+
+// Every combination of stage bits is meaningful, so the masks below name the
+// stages they cover directly: 1 is TENSOR alone, 2 GLOBAL_LOAD_ASYNC_TO_LDS
+// alone, and 9 the two of TENSOR and GLOBAL_STORE_ASYNC_FROM_LDS. A mask may
+// also name reserved stages, or every stage at once.
+
+// CHECK-LABEL: @test_masks(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 1)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 9)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2000)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2047)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 1)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 9)
+// CHECK-NEXT: ret void
+//
+void test_masks() {
+ __builtin_amdgcn_asyncmark(1);
+ __builtin_amdgcn_asyncmark(2);
+ __builtin_amdgcn_asyncmark(9);
+ __builtin_amdgcn_asyncmark(2000);
+ __builtin_amdgcn_asyncmark(2047);
+ __builtin_amdgcn_wait_asyncmark(1, 1);
+ __builtin_amdgcn_wait_asyncmark(2, 9);
}
diff --git a/llvm/docs/AMDGPUAsyncOperations.md b/llvm/docs/AMDGPUAsyncOperations.md
index 4dc01148fe781e..5715f1dc71ab55 100644
--- a/llvm/docs/AMDGPUAsyncOperations.md
+++ b/llvm/docs/AMDGPUAsyncOperations.md
@@ -16,31 +16,83 @@ internally by the compiler. A thread that initiates one or more async operations
An *asyncmark* created by a thread can be used to track async operations
initiated by that thread.
+### Stages
+
+A *stage* names a kind of async operation. Each async operation *belongs to* the
+one stage determined by the instruction that initiates it.
+
+The stages are:
+
+| Bit | Stage | Async operations |
+|---|---|---|
+| 0 | `TENSOR` | tensor loads and stores |
+| 1 | `GLOBAL_LOAD_ASYNC_TO_LDS` | global loads async to LDS |
+| 2 | `GLOBAL_LOAD_ASYNC_TO_LDS_MCAST` | multicast (cluster) global loads async to LDS |
+| 3 | `GLOBAL_STORE_ASYNC_FROM_LDS` | async global stores from LDS |
+| 5 | `BUFFER_GLOBAL_LOAD` | buffer loads to LDS and pre-gfx1250 global loads to LDS |
+
+Bits 4 and 6 through 10 are reserved for future async operations, and no
+operation belongs to them yet.
+
+Which async operations a given subtarget actually has is described in
+{ref}`AMDGPU DMA Operations <amdgpu-dma-operations>`. A stage exists on every
+subtarget that supports asyncmarks, whether or not that subtarget has any
+operation belonging to it.
+
+### Stage Masks
+
+`asyncmark` intrinsics take a *stage mask*: an 11-bit value in which a set bit
+names a stage.
+
+The zero mask, which sets no bit, is given the special meaning "every stage".
+This ensures if new stages are added that programs will continue to wait on all
+stages, if that was their intention.
+
+Bits not specified in the table above are reserved for future use. It is not an
+error to set them, but it could mean you have more conservative waits than
+necessary when the future stages are added.
+
+Setting a bit that is neither listed nor reserved is an error, and is rejected
+by the IR verifier.
+
+[Informational note --- Since the stages are independent, any mix of masks is
+valid. A program is easiest to reason about by considering the sequence of each
+stage separately.]
+
### Current Sequence
-The abstract machine maintains a sequence of asyncmarks during the execution of
-a function body, which excludes any asyncmarks produced by calls to other
-functions encountered in the currently executing function. The state of this
-sequence at each program point in the function is called the *current sequence*.
+The abstract machine maintains a separate sequence of asyncmarks *for each
+stage* during the execution of a function body, which excludes any asyncmarks
+produced by calls to other functions encountered in the currently executing
+function. The state of the sequence for a stage `S` at each program point in
+the function is called the *current sequence of* `S`.
-### `@llvm.amdgcn.asyncmark()`
+The sequences of distinct stages are independent: appending to one does not
+affect the length or contents of any other, even though multiple sequences may
+be appended to with a single call to asyncmark (by naming multiple stages).
-Produces an asyncmark and appends it to the current sequence.
+### `@llvm.amdgcn.asyncmark(i32 %K)`
-### `@llvm.amdgcn.wait.asyncmark(i16 %N)`
+Appends an asyncmark to the current sequence of every stage named in the mask
+`K`. The sequences of the stages that `K` does not name are unaffected. `K` must
+be a constant stage mask.
-Ensures that the length of the current sequence is at most `N` by removing
-asyncmarks from the start of the sequence if it is more than `N`.
+### `@llvm.amdgcn.wait.asyncmark(i16 %N, i32 %K)`
+
+For every stage named by the mask `K`, ensures that the length of the current
+sequence of that stage is at most `N` by removing asyncmarks from the start of
+that sequence if it is longer than `N`. The sequences of the stages that `K`
+does not name are unaffected. `K` must be a constant stage mask.
### Completion of Asyncmarks
-An `asyncmark()` operation `X` that produces an asyncmark `M` is
-*completed-at* a `wait.asyncmark()` operation `Y` in the same function body
-if:
+An asyncmark `M`, produced by an `asyncmark` operation `X`, is *completed-at*
+a `wait.asyncmark()` operation `Y` with mask `B` in the same function body if:
- `X` is *program-ordered* before `Y`, and
-- `M` is not in the current sequence at any operation `Z` that immediately
- follows `Y` in *program-order*.
+- `B` names the stage of the sequence that `M` belongs to, and
+- `M` is not in the current sequence at any operation `Z` that
+ immediately follows `Y` in *program-order*.
## Completion of Async Operations
@@ -49,9 +101,9 @@ not related in *program-order* with any other operations from that thread. But
the thread can use an asyncmark to ensure that the async operation is
*completed-at* some later operation.
-An async operation `A` *initiated-by* an instruction `I` is *completed-at* some
-`wait.asyncmark()` operation `Y` if there exists an `asyncmark()` operation `X`
-such that:
+An async operation `A`, *initiated-by* an instruction `I` is *completed-at* some
+`wait.asyncmark()` operation `Y` if there exists an `asyncmark()` operation `X`:
+
- `I` is *program-ordered* before `X`, and
- `X` is *completed-at* `Y`.
@@ -65,6 +117,11 @@ If `A` is *completed-at* a `wait.asyncmark()` operation `Y`, then `A`
## Examples
+Most examples below use the zero mask, so every asyncmark tracks every async
+operation and there is only one sequence to reason about. The
+{ref}`interleaved stages <amdgpu-async-interleaved-stages>` example shows what
+narrower masks add.
+
### Uneven blocks of async operations
```c++
@@ -73,7 +130,7 @@ void foo(global int *g, local int *l) {
async_load_to_lds(l, g);
async_load_to_lds(l, g);
async_load_to_lds(l, g);
- asyncmark();
+ asyncmark(0);
// second block; longer
async_load_to_lds(l, g);
@@ -81,15 +138,15 @@ void foo(global int *g, local int *l) {
async_load_to_lds(l, g);
async_load_to_lds(l, g);
async_load_to_lds(l, g);
- asyncmark();
+ asyncmark(0);
// third block; shorter
async_load_to_lds(l, g);
async_load_to_lds(l, g);
- asyncmark();
+ asyncmark(0);
// Wait for first block
- wait.asyncmark(2);
+ wait.asyncmark(2, 0);
}
```
@@ -98,30 +155,30 @@ void foo(global int *g, local int *l) {
```c++
void foo(global int *g, local int *l) {
// first block
- asyncmark();
+ asyncmark(0);
// second block
- asyncmark();
+ asyncmark(0);
// third block
- asyncmark();
+ asyncmark(0);
for (;;) {
- wait.asyncmark(2);
+ wait.asyncmark(2, 0);
// use data
// next block
- asyncmark();
+ asyncmark(0);
}
// flush one block
- wait.asyncmark(2);
+ wait.asyncmark(2, 0);
// flush one more block
- wait.asyncmark(1);
+ wait.asyncmark(1, 0);
// flush last block
- wait.asyncmark(0);
+ wait.asyncmark(0, 0);
}
```
@@ -132,22 +189,91 @@ extern void bar(); // may or may not initiate async operations
void foo(global int *g, local int *l) {
// first block
- asyncmark();
+ asyncmark(0);
// second block
- asyncmark();
+ asyncmark(0);
// function call
bar();
// third block
- asyncmark();
+ asyncmark(0);
// wait for the second block
- wait.asyncmark(1);
+ wait.asyncmark(1, 0);
// wait for the third block, including bar()
- wait.asyncmark(0);
+ wait.asyncmark(0, 0);
+}
+```
+
+(amdgpu-async-interleaved-stages)=
+### Interleaved stages
+
+Two kinds of async operation are in flight at once. Each is marked with a mask
+naming only its own stage, so each can be waited for without waiting for the
+other.
+
+```c++
+void foo(global int *g, local int *l, tensor_desc t) {
+ // Start a long tensor load and mark it in the TENSOR stage alone.
+ tensor_load_to_lds(l, t);
+ asyncmark(TENSOR);
+
+ // Start a short LDS load and mark it in its own stage alone.
+ async_load_to_lds(l, g);
+ asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // Wait for the LDS load only. The tensor load is still in flight: this wait
+ // does not name TENSOR, so it neither counts nor removes that asyncmark.
+ wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // perform synchronization / use the data loaded by async_load_to_lds
+
+ // Now wait for the tensor load.
+ wait.asyncmark(0, TENSOR);
+}
+```
+
+Had both marks used the zero mask, the first wait would have drained the tensor
+load as well, and the overlap would have been lost.
+
+The same applies to two stages tracked by one hardware counter. Here both stages
+use the async counter, but the sequences are still separate:
+
+```c++
+void foo(global int *g, local int *l) {
+ async_load_to_lds(l, g);
+ asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS); // X
+
+ async_store_from_lds(g, l);
+ asyncmark(GLOBAL_STORE_ASYNC_FROM_LDS); // Y
+
+ // Completes X. Y is in a different sequence and is not completed here, even
+ // though both stages are tracked by the same counter.
+ wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+}
+```
+
+A mask need not name just one stage. An asyncmark whose mask names several is
+appended to each of their sequences, and a wait whose mask names several trims
+each of them:
+
+```c++
+void foo(global int *g, local int *l, tensor_desc t) {
+ tensor_load_to_lds(l, t);
+ async_load_to_lds(l, g);
+
+ // One asyncmark, appended to both sequences.
+ asyncmark(TENSOR | GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // Removes the mark in GLOBAL_LOAD_ASYNC_TO_LDS. The mark in TENSOR is a
+ // separate asyncmark and stays where it is.
+ wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // Removes the mark in TENSOR.
+ wait.asyncmark(0, TENSOR);
}
```
@@ -164,15 +290,15 @@ correctness. But inlining such a call may result in redundant waits.
```c++
void foo() {
...
- asyncmark(); // X
- ... // no wait.asyncmark()
+ asyncmark(0); // X
+ ... // no wait.asyncmark()
}
void bar() {
- asyncmark(); // B
- asyncmark(); // C
+ asyncmark(0); // B
+ asyncmark(0); // C
foo();
- wait.asyncmark(1); // D
+ wait.asyncmark(1, 0); // D
}
```
@@ -182,12 +308,12 @@ being *completed-at* `D`.
```c++
void bar() {
- asyncmark(); // B
- asyncmark(); // C
+ asyncmark(0); // B
+ asyncmark(0); // C
...
- asyncmark(); // X
- ... // no wait.asyncmark()
- wait.asyncmark(1); // D
+ asyncmark(0); // X
+ ... // no wait.asyncmark()
+ wait.asyncmark(1, 0); // D
}
```
@@ -200,16 +326,16 @@ observe the current sequence of the callee.
```c++
void foo() {
- ... // no asyncmark()
- wait.asyncmark(0); // Y
+ ... // no asyncmark()
+ wait.asyncmark(0, 0); // Y
...
}
void bar() {
- asyncmark(); // B
- asyncmark(); // C
+ asyncmark(0); // B
+ asyncmark(0); // C
foo();
- wait.asyncmark(1); // D
+ wait.asyncmark(1, 0); // D
}
```
@@ -219,12 +345,12 @@ examined at `Y`.
```c++
void bar() {
- asyncmark(); // B
- asyncmark(); // C
- ... // no asyncmark()
- wait.asyncmark(0); // Y
+ asyncmark(0); // B
+ asyncmark(0); // C
+ ... // no asyncmark()
+ wait.asyncmark(0, 0); // Y
...
- wait.asyncmark(1); // D
+ wait.asyncmark(1, 0); // D
}
```
@@ -232,31 +358,36 @@ After inlining, both `B` and `C` are *completed-at* `Y`.
### Optimization
-The implementation may eliminate asyncmark/wait intrinsics in the following
-cases. These are just examples and not meant to be an exhaustive list.
+The implementation may eliminate/modify asyncmark/wait intrinsics in the
+following cases. These are just examples and not meant to be an exhaustive list.
-1. An `asyncmark` operation which remains in the current sequence along every
+1. An `asyncmark` operation which remains in a current sequence along every
path that reaches the function exit.
```c++
void foo() {
...
- asyncmark(); // X
- ... // no wait.asyncmark()
+ asyncmark(0); // X
+ ... // no wait.asyncmark()
}
```
- Here, `X` can be eliminated.
+ Here, the zero mask names every stage, and each independent mark remains, so
+ the entire mark `X` can be eliminated. If only some sequences qualified, then
+ the `asyncmark` operation could be modified to remove just those stages from
+ its mask.
2. A `wait.asyncmark` which sees an empty sequence of asyncmarks along every
path that reaches it.
```c++
void foo() {
- ... // no asyncmark()
- wait.asyncmark(0); // Y
+ ... // no asyncmark()
+ wait.asyncmark(0, 0); // Y
...
}
```
- Here, `Y` can be eliminated.
+ Here, as with `X` above, every sequence qualifies, so `Y` can be eliminated.
+ If only some sequences qualified, the `wait.asyncmark` could be modified to
+ only wait for the relevant stages.
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index ed80bc4628b6fa..b700d4a1e9cbc4 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -3020,16 +3020,21 @@ def int_amdgcn_pops_exiting_wave_id :
DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrHasSideEffects]>;
// Sets a marker in the stream of async requests. Modelled as InaccessibleMem.
+// The argument is a mask of the async stages to create a mark for.
+// A zero mask marks every stage.
def int_amdgcn_asyncmark : ClangBuiltin<"__builtin_amdgcn_asyncmark">,
- Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn,
- IntrNoCallback, IntrNoFree]>;
+ Intrinsic<[], [llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, Range<ArgIndex<0>, 0, 2048>,
+ IntrNoMem, IntrHasSideEffects, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
-// Waits until the Nth previous marker is completed, if it exists.
+// Waits until the Nth previous marker is completed in each async stage's sequence,
+// if it exists. The second argument is a mask of the stages to wait on.
+// A zero mask waits on every stage.
def int_amdgcn_wait_asyncmark :
ClangBuiltin<"__builtin_amdgcn_wait_asyncmark">,
- Intrinsic<[], [llvm_i16_ty],
- [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects,
- IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+ Intrinsic<[], [llvm_i16_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, Range<ArgIndex<1>, 0, 2048>,
+ IntrNoMem, IntrHasSideEffects, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
//===----------------------------------------------------------------------===//
// GFX10 Intrinsics
diff --git a/llvm/include/llvm/Support/AMDGPUAsyncStages.h b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
new file mode 100644
index 00000000000000..e00ac8bc63e1bd
--- /dev/null
+++ b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
@@ -0,0 +1,205 @@
+//===- AMDGPUAsyncStages.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Shared AMDGPU asyncmark stage definitions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
+#define LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+
+namespace llvm {
+namespace AMDGPU {
+
+/// Bit mask of the async stages tracked by the asyncmark / wait_asyncmark
+/// intrinsics. Each stage has its own independent sequence of marks.
+///
+/// A single stage is represented as a mask with exactly one bit set. Iterating
+/// over a mask visits each stage it names, from the least significant bit up.
+///
+/// This type also provides certain stronger guarantees than a simple integer:
+/// - Default constructor initializes the mask to zero.
+/// - Constructor ensures undefined bits cannot be set.
+class AsyncStages {
+public:
+ using value_type = uint32_t;
+
+ static constexpr unsigned NUM_STAGES = 11;
+
+ // The bit of each stage is part of the IR, so do not renumber them. Some
+ // values are RESERVED for later use.
+ enum : value_type {
+ NONE = 0,
+ // Tensor loads to LDS and tensor stores from LDS.
+ TENSOR = 1u << 0,
+ // Asynchronous global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS = 1u << 1,
+ // Asynchronous multicast (cluster) global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS_MCAST = 1u << 2,
+ // Asynchronous global stores from LDS.
+ GLOBAL_STORE_ASYNC_FROM_LDS = 1u << 3,
+ // Held for downstream use.
+ RESERVED_4 = 1u << 4,
+ // Buffer loads to LDS and pre-gfx1250 global loads to LDS.
+ BUFFER_GLOBAL_LOAD = 1u << 5,
+ RESERVED_6 = 1u << 6,
+ RESERVED_7 = 1u << 7,
+ RESERVED_8 = 1u << 8,
+ RESERVED_9 = 1u << 9,
+ RESERVED_10 = 1u << 10,
+
+ RESERVED = RESERVED_4 | RESERVED_6 | RESERVED_7 | RESERVED_8 | RESERVED_9 |
+ RESERVED_10,
+
+ // Bits that a mask may legally set. Reserved stages are included: naming a
+ // stage whose operations do not exist yet is harmless, and accepting the
+ // bit keeps masks portable as stages are filled in.
+ ALL = (1u << NUM_STAGES) - 1
+ };
+
+ /// Iterates over the stages named by a mask.
+ /// NOLINTNEXTLINE
+ class const_iterator
+ : public iterator_facade_base<const_iterator, std::forward_iterator_tag,
+ AsyncStages> {
+ // The "end" iterator is also the default-constructed iterator.
+ // We naturally move towards the "end" by clearing the set bits from least
+ // to most significant.
+ AsyncStages::value_type Cur = 0;
+
+ public:
+ const_iterator() = default;
+ const_iterator(AsyncStages S) : Cur(S.value()) {}
+
+ bool operator==(const const_iterator &Other) const {
+ return Cur == Other.Cur;
+ }
+
+ AsyncStages operator*() const {
+ // Return only rightmost (least significant) bit set.
+ return Cur ? (Cur & (1u << countr_zero(Cur))) : 0;
+ }
+
+ const_iterator &operator++() {
+ // Keep all bits except the least significant bit set.
+ Cur &= maskTrailingZeros<AsyncStages::value_type>(countr_zero(Cur) + 1);
+ return *this;
+ }
+ };
+
+ constexpr AsyncStages() = default;
+ constexpr AsyncStages(value_type V) : Data(V) {
+ assert((V & ALL) == V && "Bits set out of bounds!");
+ }
+
+ /// \returns true if \p Imm is a legal stage mask operand of the asyncmark /
+ /// wait_asyncmark intrinsics and pseudos.
+ static constexpr bool isValidMask(int64_t Imm) {
+ return Imm >= 0 && (Imm & ~int64_t(ALL)) == 0;
+ }
+
+ /// \returns the stages named by the stage mask operand \p Imm of the
+ /// asyncmark / wait_asyncmark intrinsics and pseudos. The zero mask is the
+ /// one exception to a set bit naming a stage: it names every stage rather
+ /// than none.
+ static constexpr AsyncStages fromMaskOperand(value_type Imm) {
+ return Imm ? AsyncStages(Imm) : AsyncStages(ALL);
+ }
+
+ constexpr unsigned size() const { return popcount(Data); }
+ constexpr bool any() const { return Data != 0; }
+ constexpr bool none() const { return Data == 0; }
+ constexpr value_type value() const { return Data; }
+
+ explicit constexpr operator bool() const { return any(); }
+
+ const_iterator begin() const { return *this; }
+ const_iterator end() const { return {}; }
+
+ /// \returns the position of this single stage in a mask, for indexing
+ /// per-stage data.
+ unsigned index() const {
+ assert(size() == 1 && "Expected a single stage");
+ return countr_zero(Data);
+ }
+
+ /// \returns the name of this single stage.
+ const char *getName() const {
+ switch (Data) {
+ case TENSOR:
+ return "TENSOR";
+ case GLOBAL_LOAD_ASYNC_TO_LDS:
+ return "GLOBAL_LOAD_ASYNC_TO_LDS";
+ case GLOBAL_LOAD_ASYNC_TO_LDS_MCAST:
+ return "GLOBAL_LOAD_ASYNC_TO_LDS_MCAST";
+ case GLOBAL_STORE_ASYNC_FROM_LDS:
+ return "GLOBAL_STORE_ASYNC_FROM_LDS";
+ case RESERVED_4:
+ return "RESERVED_4";
+ case BUFFER_GLOBAL_LOAD:
+ return "BUFFER_GLOBAL_LOAD";
+ case RESERVED_6:
+ return "RESERVED_6";
+ case RESERVED_7:
+ return "RESERVED_7";
+ case RESERVED_8:
+ return "RESERVED_8";
+ case RESERVED_9:
+ return "RESERVED_9";
+ case RESERVED_10:
+ return "RESERVED_10";
+ }
+ llvm_unreachable("Expected a single stage");
+ }
+
+ constexpr AsyncStages operator&(AsyncStages Other) const {
+ return Data & Other.Data;
+ }
+ constexpr AsyncStages operator-(AsyncStages Other) const {
+ return Data & ~Other.Data;
+ }
+
+ constexpr bool operator==(AsyncStages Other) const {
+ return Data == Other.Data;
+ }
+
+ constexpr AsyncStages &operator|=(AsyncStages Other) {
+ Data |= Other.Data;
+ return *this;
+ }
+
+private:
+ value_type Data = NONE;
+};
+
+/// Print the stages named by \p S as a '|'-separated list, or "all" if it
+/// names every stage.
+inline raw_ostream &operator<<(raw_ostream &OS, AsyncStages S) {
+ if (S == AsyncStages::ALL)
+ return OS << "all";
+ if (S.none())
+ return OS << "none";
+ interleave(S, OS, [&OS](AsyncStages Stage) { OS << Stage.getName(); }, "|");
+ return OS;
+}
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index fba620adb463d8..471d59f82d0637 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1761,6 +1761,17 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
switch (F->getIntrinsicID()) {
default:
break;
+ // Legacy asyncmark intrinsics without the stage operand.
+ case Intrinsic::amdgcn_asyncmark:
+ case Intrinsic::amdgcn_wait_asyncmark: {
+ Intrinsic::ID ID = F->getIntrinsicID();
+ if (F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
+ rename(F);
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
+ return true;
+ }
+ break;
+ }
// Legacy wmma iu intrinsics without the optional clamp operand.
case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
if (F->arg_size() == 7) {
@@ -6452,6 +6463,24 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
break;
}
+ case Intrinsic::amdgcn_asyncmark:
+ case Intrinsic::amdgcn_wait_asyncmark: {
+ // Legacy asyncmark intrinsics missed the stage mask operand. Append a zero
+ // mask, which names every stage and so preserves the behavior they had.
+ SmallVector<Value *, 2> Args(CI->args());
+ Args.push_back(Builder.getInt32(0));
+
+ SmallVector<OperandBundleDef, 1> Bundles;
+ CI->getOperandBundlesAsDefs(Bundles);
+
+ NewCall = Builder.CreateCall(NewFn, Args, Bundles);
+ NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ NewCall->setCallingConv(CI->getCallingConv());
+ NewCall->setAttributes(CI->getAttributes());
+ NewCall->copyMetadata(*CI);
+ break;
+ }
+
case Intrinsic::thread_pointer: {
NewCall = Builder.CreateCall(NewFn, {});
break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index b391761e3b9062..325ba441769e1a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -30,6 +30,7 @@
#include "llvm/MC/MCObjectStreamer.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Format.h"
@@ -375,15 +376,25 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
}
if (MI->getOpcode() == AMDGPU::ASYNCMARK) {
- if (isVerbose())
- OutStreamer->emitRawComment(" asyncmark");
+ if (isVerbose()) {
+ std::string Comment;
+ raw_string_ostream OS(Comment);
+ OS << " asyncmark(stages="
+ << AMDGPU::AsyncStages::fromMaskOperand(MI->getOperand(0).getImm())
+ << ')';
+ OutStreamer->emitRawComment(Comment);
+ }
return;
}
if (MI->getOpcode() == AMDGPU::WAIT_ASYNCMARK) {
if (isVerbose()) {
- OutStreamer->emitRawComment(" wait_asyncmark(" +
- Twine(MI->getOperand(0).getImm()) + ")");
+ std::string Comment;
+ raw_string_ostream OS(Comment);
+ OS << " wait_asyncmark(" << MI->getOperand(0).getImm() << ", stages="
+ << AMDGPU::AsyncStages::fromMaskOperand(MI->getOperand(1).getImm())
+ << ')';
+ OutStreamer->emitRawComment(Comment);
}
return;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
index 0e0d8ba7d13d46..dbac8bf66dfdef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
@@ -13,6 +13,8 @@
#include "AMDGPUMIRFormatter.h"
#include "SIMachineFunctionInfo.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
+#include "llvm/Support/Format.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -37,6 +39,11 @@ StringLiteral LgkmcntName = "Lgkmcnt";
StringLiteral LoadcntName = "Loadcnt";
StringLiteral DscntName = "Dscnt";
+const char AsyncStageImmPrefix = '.';
+constexpr StringLiteral AsyncStageDelim(".");
+constexpr StringLiteral ReservedStagePrefix("RES");
+constexpr StringLiteral AllStages("AllStages");
+
void AMDGPUMIRFormatter::printSWaitAluImm(uint64_t Imm, raw_ostream &OS) const {
bool NonePrinted = true;
ListSeparator Delim(SWaitAluDelim);
@@ -109,10 +116,44 @@ void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(uint64_t Imm,
OS << AllOff;
}
+void AMDGPUMIRFormatter::printAsyncStageMaskImm(int64_t Imm,
+ raw_ostream &OS) const {
+ if (!AMDGPU::AsyncStages::isValidMask(Imm)) {
+ OS << Imm;
+ return;
+ }
+
+ OS << AsyncStageImmPrefix;
+ AMDGPU::AsyncStages Mask(Imm);
+ // The zero mask names every stage. Spell that out rather than printing an
+ // empty list. The saturated mask means the same thing but is a different
+ // immediate, so it keeps its spelled-out list to stay round-trippable.
+ if (Mask.none()) {
+ OS << AllStages;
+ return;
+ }
+
+ // Reserved stages have no meaningful name, collect them into a single
+ // hex bitmask instead.
+ ListSeparator Delim(AsyncStageDelim);
+ for (AMDGPU::AsyncStages S : Mask - AMDGPU::AsyncStages::RESERVED)
+ OS << Delim << S.getName();
+ if (AMDGPU::AsyncStages Reserved = Mask & AMDGPU::AsyncStages::RESERVED)
+ OS << Delim << ReservedStagePrefix
+ << format_hex_no_prefix(Reserved.value(), 1);
+}
+
void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI,
std::optional<unsigned int> OpIdx, int64_t Imm) const {
switch (MI.getOpcode()) {
+ case AMDGPU::ASYNCMARK:
+ case AMDGPU::WAIT_ASYNCMARK:
+ if (OpIdx == MI.getNumExplicitOperands() - 1u)
+ printAsyncStageMaskImm(Imm, OS);
+ else
+ MIRFormatter::printImm(OS, MI, OpIdx, Imm);
+ break;
case AMDGPU::S_WAITCNT:
case AMDGPU::S_WAITCNT_soft:
printSWaitcntImm(Imm, OS);
@@ -151,6 +192,14 @@ bool AMDGPUMIRFormatter::parseImmMnemonic(const unsigned OpCode,
return parseSWaitAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
case AMDGPU::S_DELAY_ALU:
return parseSDelayAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
+ case AMDGPU::ASYNCMARK:
+ if (OpIdx == 0)
+ return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
+ break;
+ case AMDGPU::WAIT_ASYNCMARK:
+ if (OpIdx == 1)
+ return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
+ break;
default:
break;
}
@@ -488,6 +537,58 @@ bool AMDGPUMIRFormatter::parseSDelayAluImmMnemonic(
return false;
}
+// Parse the async stage mask of asyncmark/wait_asyncmark. The mnemonic names
+// the stages the mask names.
+bool AMDGPUMIRFormatter::parseAsyncStageMaskImmMnemonic(
+ const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
+ MIRFormatter::ErrorCallbackType &ErrorCallback) const {
+ if (!Src.consumeInteger(10, Imm))
+ return false;
+
+ if (!Src.consume_front(AsyncStageImmPrefix))
+ return ErrorCallback(Src.begin(), "expected prefix");
+ if (Src.empty())
+ return ErrorCallback(Src.begin(), "expected <StageName>");
+
+ if (Src == AllStages) {
+ Imm = 0;
+ return false;
+ }
+
+ AMDGPU::AsyncStages StageMask;
+ while (!Src.empty()) {
+ StringRef Name = Src.substr(0, Src.find(AsyncStageDelim));
+ StringRef::iterator NamePos = Src.begin();
+ Src.consume_front(Name);
+ Src.consume_front(AsyncStageDelim);
+
+ // The reserved stages arrive together as one hex bitmask.
+ StringRef Bits = Name;
+ if (Bits.consume_front(ReservedStagePrefix)) {
+ uint32_t Mask;
+ if (Bits.getAsInteger(16, Mask) || Bits.empty() || !Mask)
+ return ErrorCallback(NamePos, "invalid async stage mask");
+ if (!AMDGPU::AsyncStages::isValidMask(Mask))
+ return ErrorCallback(NamePos, "async stage mask out of range");
+ if (Mask & ~AMDGPU::AsyncStages::RESERVED)
+ return ErrorCallback(NamePos,
+ "async stage mask names a non-reserved stage");
+ StageMask |= Mask;
+ continue;
+ }
+
+ auto Stages = AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL);
+ auto It = find_if(
+ Stages, [Name](AMDGPU::AsyncStages S) { return Name == S.getName(); });
+ if (It == Stages.end())
+ return ErrorCallback(NamePos, "invalid async stage name");
+ StageMask |= *It;
+ }
+
+ Imm = StageMask.value();
+ return false;
+}
+
bool AMDGPUMIRFormatter::parseCustomPseudoSourceValue(
StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS,
const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h
index 464d48c17a1f8e..8396dee32239dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.h
@@ -58,6 +58,8 @@ class AMDGPUMIRFormatter final : public MIRFormatter {
void printSWaitLoadcntDscntImm(uint64_t Imm, raw_ostream &OS) const;
/// Print the string to represent s_delay_alu immediate value
void printSDelayAluImm(int64_t Imm, llvm::raw_ostream &OS) const;
+ /// Prints the string to represent an asyncmark stage mask.
+ void printAsyncStageMaskImm(int64_t Imm, raw_ostream &OS) const;
/// Parse the immediate pseudo literal for s_wait_alu
bool parseSWaitAluImmMnemonic(
@@ -79,6 +81,10 @@ class AMDGPUMIRFormatter final : public MIRFormatter {
const unsigned int OpIdx, int64_t &Imm, llvm::StringRef &Src,
llvm::MIRFormatter::ErrorCallbackType &ErrorCallback) const;
+ /// Parse the immediate pseudo literal for an asyncmark stage mask
+ bool parseAsyncStageMaskImmMnemonic(
+ const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
+ MIRFormatter::ErrorCallbackType &ErrorCallback) const;
};
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bd2815d0491b5b..bb8f88dcc097ea 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -422,15 +423,22 @@ class SIInsertWaitcnts {
return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
}
- bool shouldUpdateAsyncMark(const MachineInstr &MI,
- AMDGPU::InstCounterType T) const {
+ // The stage whose pending async score an event on counter \p T from \p MI
+ // updates, if any.
+ AMDGPU::AsyncStages getAsyncStageToUpdate(const MachineInstr &MI,
+ AMDGPU::InstCounterType T) const {
+ AMDGPU::InstCounterType StageCounter;
if (SIInstrInfo::usesTENSOR_CNT(MI))
- return T == AMDGPU::TENSOR_CNT;
- if (!isAsyncLdsDmaWrite(MI))
- return false;
- if (SIInstrInfo::usesASYNC_CNT(MI))
- return T == AMDGPU::ASYNC_CNT;
- return T == AMDGPU::LOAD_CNT;
+ StageCounter = AMDGPU::TENSOR_CNT;
+ else if (!isAsyncLdsDmaWrite(MI))
+ return AMDGPU::AsyncStages::NONE;
+ else if (SIInstrInfo::usesASYNC_CNT(MI))
+ StageCounter = AMDGPU::ASYNC_CNT;
+ else
+ StageCounter = AMDGPU::LOAD_CNT;
+ if (T != StageCounter)
+ return AMDGPU::AsyncStages::NONE;
+ return SIInstrInfo::getAsyncStage(MI);
}
bool isVmemAccess(const MachineInstr &MI) const;
@@ -580,14 +588,14 @@ class WaitcntBrackets {
MCRegister Reg) const;
void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
AMDGPU::Waitcnt &Wait) const;
- AMDGPU::Waitcnt determineAsyncWait(unsigned N);
+ AMDGPU::Waitcnt determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages);
void tryClearSCCWriteEvent(MachineInstr *Inst);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
void updateByEvent(HWEvents E, MachineInstr &MI);
- void recordAsyncMark(MachineInstr &MI);
+ void recordAsyncMark(MachineInstr &MI, AMDGPU::AsyncStages Stages);
HWEvents getPendingEvents() const { return PendingEvents; }
bool hasPendingEvent() const { return PendingEvents.any(); }
@@ -690,7 +698,8 @@ class WaitcntBrackets {
static bool mergeScore(const MergeInfo &M, unsigned &Score,
unsigned OtherScore);
- bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
+ bool mergeAsyncMarks(AMDGPU::AsyncStages Stage,
+ ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks);
iterator_range<MCRegUnitIterator> regunits(MCRegister Reg) const {
@@ -811,8 +820,9 @@ class WaitcntBrackets {
// alias info. One store is kept per unique AAInfo.
SmallVector<const MachineInstr *> LDSDMAStores;
- // State of all counters at each async mark encountered so far.
- SmallVector<CounterValueArray> AsyncMarks;
+ // State of all counters at each async mark encountered so far, per stage.
+ std::array<SmallVector<CounterValueArray, 0>, AMDGPU::AsyncStages::NUM_STAGES>
+ AsyncMarks;
// But in the rare pathological case, a nest of loops that pushes marks
// without waiting on any mark can cause AsyncMarks to grow very large. We cap
@@ -821,8 +831,8 @@ class WaitcntBrackets {
static constexpr unsigned MaxAsyncMarks = 16;
// Track the upper bound score for async operations that are not part of a
- // mark yet. Initialized to all zeros.
- CounterValueArray AsyncScore{};
+ // mark yet, per stage. Initialized to all zeros.
+ std::array<CounterValueArray, AMDGPU::AsyncStages::NUM_STAGES> AsyncScore{};
};
SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
@@ -1091,9 +1101,8 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
setVMemScore(LDSDMA_BEGIN + static_cast<VMEMID>(Slot), T, CurrScore);
}
- if (Context->shouldUpdateAsyncMark(Inst, T)) {
- AsyncScore[T] = CurrScore;
- }
+ if (AMDGPU::AsyncStages Stage = Context->getAsyncStageToUpdate(Inst, T))
+ AsyncScore[Stage.index()][T] = CurrScore;
if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
setRegScore(AMDGPU::SCC, T, CurrScore);
@@ -1102,20 +1111,28 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
}
}
-void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
+void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
+ AMDGPU::AsyncStages Stages) {
// In the absence of loops, AsyncMarks can grow linearly with the program
// until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
// limit every time we push a new mark, but that seems like unnecessary work
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
- AsyncMarks.push_back(AsyncScore);
- LLVM_DEBUG({
- dbgs() << "recordAsyncMark:\n" << Inst;
- for (const auto &Mark : AsyncMarks) {
- llvm::interleaveComma(Mark, dbgs());
- dbgs() << '\n';
- }
- });
+ //
+ // A mark is inserted in every stage named. Each stage gets a copy of its
+ // corresponding score.
+ LLVM_DEBUG(dbgs() << "recordAsyncMark(stages=" << Stages << "):\n" << Inst);
+ for (AMDGPU::AsyncStages S : Stages) {
+ AsyncMarks[S.index()].push_back(AsyncScore[S.index()]);
+ LLVM_DEBUG({
+ dbgs() << " stage " << S << ":\n";
+ for (const auto &Mark : AsyncMarks[S.index()]) {
+ dbgs() << " ";
+ llvm::interleaveComma(Mark, dbgs());
+ dbgs() << '\n';
+ }
+ });
+ }
}
void WaitcntBrackets::print(raw_ostream &OS) const {
@@ -1218,55 +1235,59 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
}
OS << '\n';
- OS << "Async score: ";
- if (AsyncScore.empty())
- OS << "none";
- else
- llvm::interleaveComma(AsyncScore, OS);
- OS << '\n';
+ for (AMDGPU::AsyncStages S : AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL)) {
+ const CounterValueArray &Score = AsyncScore[S.index()];
+ OS << "Async score (stage " << S << "): ";
+ if (llvm::all_of(Score, [](unsigned V) { return V == 0; }))
+ OS << "none";
+ else
+ llvm::interleaveComma(Score, OS);
+ OS << '\n';
- OS << "Async marks: " << AsyncMarks.size() << '\n';
+ OS << "Async marks (stage " << S << "): " << AsyncMarks[S.index()].size()
+ << '\n';
- for (const auto &Mark : AsyncMarks) {
- for (auto T : AMDGPU::inst_counter_types()) {
- unsigned MarkedScore = Mark[T];
- switch (T) {
- case AMDGPU::LOAD_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::DS_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::EXP_CNT:
- OS << " EXP_CNT: " << MarkedScore;
- break;
- case AMDGPU::STORE_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::SAMPLE_CNT:
- OS << " SAMPLE_CNT: " << MarkedScore;
- break;
- case AMDGPU::BVH_CNT:
- OS << " BVH_CNT: " << MarkedScore;
- break;
- case AMDGPU::KM_CNT:
- OS << " KM_CNT: " << MarkedScore;
- break;
- case AMDGPU::X_CNT:
- OS << " X_CNT: " << MarkedScore;
- break;
- case AMDGPU::ASYNC_CNT:
- OS << " ASYNC_CNT: " << MarkedScore;
- break;
- default:
- OS << " UNKNOWN: " << MarkedScore;
- break;
+ for (const auto &Mark : AsyncMarks[S.index()]) {
+ for (auto T : AMDGPU::inst_counter_types()) {
+ unsigned MarkedScore = Mark[T];
+ switch (T) {
+ case AMDGPU::LOAD_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::DS_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::EXP_CNT:
+ OS << " EXP_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::STORE_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::SAMPLE_CNT:
+ OS << " SAMPLE_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::BVH_CNT:
+ OS << " BVH_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::KM_CNT:
+ OS << " KM_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::X_CNT:
+ OS << " X_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::ASYNC_CNT:
+ OS << " ASYNC_CNT: " << MarkedScore;
+ break;
+ default:
+ OS << " UNKNOWN: " << MarkedScore;
+ break;
+ }
}
+ OS << '\n';
}
- OS << '\n';
}
OS << '\n';
}
@@ -1387,44 +1408,54 @@ void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
}
}
-AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
- LLVM_DEBUG({
- dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
- << ":\n";
- for (const auto &Mark : AsyncMarks) {
- llvm::interleaveComma(Mark, dbgs());
- dbgs() << '\n';
- }
- });
+AMDGPU::Waitcnt
+WaitcntBrackets::determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages) {
+ // Trim each stage the wait names down to at most N marks. The waits
+ // implied by the marks dropped from each stage accumulate into one Waitcnt,
+ // so a wait covering several stages is the union of their requirements.
+ AMDGPU::Waitcnt Wait;
+ for (AMDGPU::AsyncStages S : Stages) {
+ auto &StageMarks = AsyncMarks[S.index()];
+ LLVM_DEBUG({
+ dbgs() << "Need " << N << " async marks in stage " << S << ". Found "
+ << StageMarks.size() << ":\n";
+ for (const auto &Mark : StageMarks) {
+ llvm::interleaveComma(Mark, dbgs());
+ dbgs() << '\n';
+ }
+ });
- if (AsyncMarks.size() == MaxAsyncMarks) {
- // Enforcing MaxAsyncMarks here is unnecessary work because the size of
- // MaxAsyncMarks is linear when traversing straightline code. But we do
- // need to check if truncation may have occured at a merge, and adjust N
- // to ensure that a wait is generated.
- LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
- N = std::min(N, (unsigned)MaxAsyncMarks - 1);
- }
+ unsigned StageN = N;
+ if (StageMarks.size() == MaxAsyncMarks) {
+ // Enforcing MaxAsyncMarks here is unnecessary work because the size of
+ // MaxAsyncMarks is linear when traversing straightline code. But we do
+ // need to check if truncation may have occured at a merge, and adjust N
+ // to ensure that a wait is generated.
+ LLVM_DEBUG(
+ dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
+ StageN = std::min(StageN, (unsigned)MaxAsyncMarks - 1);
+ }
- AMDGPU::Waitcnt Wait;
- if (AsyncMarks.size() <= N) {
- LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
- return Wait;
- }
+ if (StageMarks.size() <= StageN) {
+ LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
+ continue;
+ }
- size_t MarkIndex = AsyncMarks.size() - N - 1;
- const auto &RequiredMark = AsyncMarks[MarkIndex];
- for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
- determineWaitForScore(T, RequiredMark[T], Wait);
+ size_t MarkIndex = StageMarks.size() - StageN - 1;
+ const auto &RequiredMark = StageMarks[MarkIndex];
+ for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
+ determineWaitForScore(T, RequiredMark[T], Wait);
+ }
- // Immediately remove the waited mark and all older ones
- // This happens BEFORE the wait is actually inserted, which is fine
- // because we've already extracted the wait requirements
- LLVM_DEBUG({
- dbgs() << "Removing " << (MarkIndex + 1)
- << " async marks after determining wait\n";
- });
- AsyncMarks.erase(AsyncMarks.begin(), AsyncMarks.begin() + MarkIndex + 1);
+ // Immediately remove the waited mark and all older ones
+ // This happens BEFORE the wait is actually inserted, which is fine
+ // because we've already extracted the wait requirements
+ LLVM_DEBUG({
+ dbgs() << "Removing " << (MarkIndex + 1)
+ << " async marks after determining wait\n";
+ });
+ StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
+ }
LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
return Wait;
@@ -1719,7 +1750,8 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
} else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
- AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
+ AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
+ N, AMDGPU::AsyncStages::fromMaskOperand(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
@@ -2016,7 +2048,8 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
// Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
// shows up in the assembly as a comment with the original parameter N.
unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
- AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
+ AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
+ N, AMDGPU::AsyncStages::fromMaskOperand(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
std::optional<AMDGPU::InstCounterType> CT =
@@ -2779,37 +2812,39 @@ bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
return OtherShifted > MyShifted;
}
-bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
+bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStages Stage,
+ ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks) {
bool StrictDom = false;
+ auto &StageMarks = AsyncMarks[Stage.index()];
- LLVM_DEBUG(dbgs() << "Merging async marks ...");
+ LLVM_DEBUG(dbgs() << "Merging async marks of stage " << Stage << " ...");
// Early exit: nothing to merge when both sides are empty.
- if (AsyncMarks.empty() && OtherMarks.empty()) {
+ if (StageMarks.empty() && OtherMarks.empty()) {
LLVM_DEBUG(dbgs() << " nothing to merge\n");
return false;
}
LLVM_DEBUG(dbgs() << '\n');
// Determine maximum length needed after merging
- auto MaxSize = (unsigned)std::max(AsyncMarks.size(), OtherMarks.size());
+ auto MaxSize = (unsigned)std::max(StageMarks.size(), OtherMarks.size());
MaxSize = std::min(MaxSize, MaxAsyncMarks);
// Keep only the most recent marks within our limit.
- if (AsyncMarks.size() > MaxSize)
- AsyncMarks.erase(AsyncMarks.begin(),
- AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
+ if (StageMarks.size() > MaxSize)
+ StageMarks.erase(StageMarks.begin(),
+ StageMarks.begin() + (StageMarks.size() - MaxSize));
// Pad with zero-filled marks if our list is shorter. Zero represents "no
// pending async operations at this checkpoint" and acts as the identity
// element for max() during merging. We pad at the beginning since the marks
// need to be aligned in most-recent order.
constexpr CounterValueArray ZeroMark{};
- AsyncMarks.insert(AsyncMarks.begin(), MaxSize - AsyncMarks.size(), ZeroMark);
+ StageMarks.insert(StageMarks.begin(), MaxSize - StageMarks.size(), ZeroMark);
LLVM_DEBUG({
dbgs() << "Before merge:\n";
- for (const auto &Mark : AsyncMarks) {
+ for (const auto &Mark : StageMarks) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
@@ -2828,26 +2863,26 @@ bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
// re-expressed in the new frame instead of being left with a stale
// (pre-merge) score.
const size_t OtherSize = OtherMarks.size();
- const size_t OurSize = AsyncMarks.size();
- // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
- // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
+ const size_t OurSize = StageMarks.size();
+ // After the both-empty early-return above, max(StageMarks, OtherMarks) >= 1,
+ // and the erase/pad steps normalize StageMarks to exactly MaxSize. Hence
// OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
// seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
// the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
assert(OurSize >= 1 &&
- "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
+ "StageMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
for (auto Idx : seq_inclusive<size_t>(1, OurSize)) {
const CounterValueArray &OtherMark =
Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
for (auto T : inst_counter_types(Context->MaxCounter))
StrictDom |=
- mergeScore(MergeInfos[T], AsyncMarks[OurSize - Idx][T], OtherMark[T]);
+ mergeScore(MergeInfos[T], StageMarks[OurSize - Idx][T], OtherMark[T]);
}
LLVM_DEBUG({
dbgs() << "After merge:\n";
- for (const auto &Mark : AsyncMarks) {
+ for (const auto &Mark : StageMarks) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
@@ -2938,9 +2973,12 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
- StrictDom |= mergeAsyncMarks(MergeInfos, Other.AsyncMarks);
- for (auto T : inst_counter_types(Context->MaxCounter))
- StrictDom |= mergeScore(MergeInfos[T], AsyncScore[T], Other.AsyncScore[T]);
+ for (AMDGPU::AsyncStages S : AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL)) {
+ StrictDom |= mergeAsyncMarks(S, MergeInfos, Other.AsyncMarks[S.index()]);
+ for (auto T : inst_counter_types(Context->MaxCounter))
+ StrictDom |= mergeScore(MergeInfos[T], AsyncScore[S.index()][T],
+ Other.AsyncScore[S.index()][T]);
+ }
purgeEmptyTrackingData();
return StrictDom;
@@ -3118,7 +3156,8 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// Asyncmarks record the current wait state and so should not allow
// waitcnts that occur after them to be merged into waitcnts that occur
// before.
- ScoreBrackets.recordAsyncMark(Inst);
+ ScoreBrackets.recordAsyncMark(Inst, AMDGPU::AsyncStages::fromMaskOperand(
+ Inst.getOperand(0).getImm()));
continue;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9b81ed62227ad1..5ff8ebcca7fcca 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -32,6 +32,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/MC/MCContext.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Target/TargetMachine.h"
#include <tuple>
@@ -4816,6 +4817,50 @@ bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
return false;
}
+AMDGPU::AsyncStages SIInstrInfo::getAsyncStage(const MachineInstr &MI) {
+ if (!isLDSDMA(MI))
+ return AMDGPU::AsyncStages::NONE;
+ if (usesTENSOR_CNT(MI))
+ return AMDGPU::AsyncStages::TENSOR;
+ if (!usesASYNC_CNT(MI))
+ return AMDGPU::AsyncStages::BUFFER_GLOBAL_LOAD;
+
+ // TODO: Declare the stage of each async instruction in TableGen alongside
+ // its definition, instead of listing opcodes here.
+ switch (MI.getOpcode()) {
+ // Keep in sync with FLATInstructions.td.
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ return AMDGPU::AsyncStages::GLOBAL_LOAD_ASYNC_TO_LDS;
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ return AMDGPU::AsyncStages::GLOBAL_LOAD_ASYNC_TO_LDS_MCAST;
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR:
+ return AMDGPU::AsyncStages::GLOBAL_STORE_ASYNC_FROM_LDS;
+ default:
+ llvm_unreachable("Async opcode has no associated async stage");
+ }
+}
+
bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
// Skip the full operand and register alias search modifiesRegister
// does. There's only a handful of instructions that touch this, it's only an
@@ -5875,6 +5920,14 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
}
+ if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
+ int64_t Mask = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
+ if (!AMDGPU::AsyncStages::isValidMask(Mask)) {
+ ErrInfo = "invalid asyncmark stage mask";
+ return false;
+ }
+ }
+
// Verify misc. restrictions on specific instructions.
if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 72e9cda785036a..7c7070d06b2e5d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -38,6 +38,10 @@ class MCRegisterClass;
using TargetRegisterClass = MCRegisterClass;
class ScheduleHazardRecognizer;
+namespace AMDGPU {
+class AsyncStages;
+} // namespace AMDGPU
+
constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
/// Mark the MMO of a uniform load if there are no potentially clobbering stores
@@ -875,6 +879,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
}
+ /// \returns the asyncmark stage that the async operation initiated by \p MI
+ /// belongs to, or no stage if \p MI is not an LDS DMA instruction.
+ static AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI);
+
static bool isSBarrierSCCWrite(unsigned Opcode) {
return Opcode == AMDGPU::S_BARRIER_LEAVE ||
Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 9b8649341f7488..7cf96825c3d9ee 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1728,13 +1728,13 @@ def S_WAITCNT_lds_direct : SPseudoInstSI<(outs), (ins)> {
}
let SubtargetPredicate = HasAsyncMark in {
-def ASYNCMARK : SPseudoInstSI<(outs), (ins),
- [(int_amdgcn_asyncmark)]> {
+def ASYNCMARK : SPseudoInstSI<(outs), (ins i32imm:$stage),
+ [(int_amdgcn_asyncmark timm:$stage)]> {
let maybeAtomic = 0;
let isMeta = 1;
}
-def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16), "$simm16",
- [(int_amdgcn_wait_asyncmark timm:$simm16)]> {
+def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16, i32imm:$stage),
+ "$simm16", [(int_amdgcn_wait_asyncmark timm:$simm16, timm:$stage)]> {
let maybeAtomic = 0;
let Size = 0;
let isMeta = 1;
diff --git a/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
new file mode 100644
index 00000000000000..6cd1691b522230
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
@@ -0,0 +1,58 @@
+; RUN: split-file %s %t
+; RUN: llvm-as < %t/legacy.ll | llvm-dis | FileCheck %s --check-prefix=LEGACY
+; RUN: llvm-as < %t/legacy-declared.ll | llvm-dis | FileCheck %s --check-prefix=DECLARED
+; RUN: llvm-as < %t/staged.ll | llvm-dis | FileCheck %s --check-prefix=STAGED
+
+; The asyncmark intrinsics originally had no stage mask operand. Upgrade them to
+; a zero mask, which names every stage and so is the behavior they had.
+
+;--- legacy.ll
+define void @legacy() {
+; LEGACY-LABEL: define void @legacy(
+; LEGACY-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
+; LEGACY-NEXT: ret void
+;
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ ret void
+}
+
+;--- legacy-declared.ll
+; The old declarations are spelled out and each is called more than once. The
+; intrinsics are not overloaded, so the upgraded declaration has the same name
+; as the old one.
+declare void @llvm.amdgcn.asyncmark()
+declare void @llvm.amdgcn.wait.asyncmark(i16)
+
+define void @legacy_declared() {
+; DECLARED-LABEL: define void @legacy_declared(
+; DECLARED-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; DECLARED-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; DECLARED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+; DECLARED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+; DECLARED-NEXT: ret void
+;
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ ret void
+}
+; DECLARED-NOT: .old
+
+;--- staged.ll
+; Calls that already carry a mask are left alone. The masks here are non-empty
+; so that leaving them alone is distinguishable from upgrading them.
+define void @staged() {
+; STAGED-LABEL: define void @staged(
+; STAGED-NEXT: call void @llvm.amdgcn.asyncmark(i32 1)
+; STAGED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 1)
+; STAGED-NEXT: ret void
+;
+ call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 1)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
index 2d096c5af4cc57..1a619048f5fb30 100644
--- a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
@@ -9,23 +9,23 @@ define float @raw.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %lds
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -37,23 +37,23 @@ define float @raw.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace(3)
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -65,23 +65,23 @@ define float @struct.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -93,23 +93,23 @@ define float @struct.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -124,13 +124,13 @@ define float @struct.ptr.buffer.load.vgpr.lds(ptr addrspace(8) inreg %rsrc, ptr
; CHECK-NEXT: s_mov_b32 m0, s4
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v1, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index be3339be95916e..d13af58c0f14e6 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -24,17 +24,17 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; SDAG-NEXT: v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
; SDAG-NEXT: ; wave barrier
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_b32 v15, v[0:1], off offset:8
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_b32 v16, v[8:9], off offset:48
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v17, v2 offset:84
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
@@ -65,17 +65,17 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
; GISEL-NEXT: ; wave barrier
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_b32 v15, v[0:1], off offset:8
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_b32 v16, v[8:9], off offset:48
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v17, v2 offset:84
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
@@ -96,7 +96,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
call void @llvm.amdgcn.wave.barrier()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load
%foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
@@ -108,15 +108,15 @@ entry:
call void @llvm.amdgcn.wave.barrier()
%bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
%bar_v12 = load i32, ptr addrspace(1) %bar_gep12
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds_gep21
; Wait for the next async mark.
; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds_gep22
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -151,10 +151,10 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b32 s7, s2
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[0:1] offset:4 nv
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: .LBB1_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
@@ -163,8 +163,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_add_co_i32 s7, s7, 4
; SDAG-NEXT: s_cmp_lt_i32 s6, s3
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: ds_load_b32 v3, v3
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
@@ -173,14 +173,14 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_add_co_i32 s0, s0, -8
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; SDAG-NEXT: ds_load_b32 v0, v0
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
@@ -206,19 +206,19 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: s_mov_b32 s8, s2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: .LBB1_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
; GISEL-NEXT: s_add_co_i32 s7, s7, 1
; GISEL-NEXT: v_dual_mov_b32 v1, s9 :: v_dual_mov_b32 v2, s8
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v2, v2
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -231,7 +231,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_lshl_b32 s0, s3, 2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_add_co_u32 s0, s2, s0
; GISEL-NEXT: v_mov_b32_e32 v1, 0
@@ -240,7 +240,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GISEL-NEXT: ds_load_b32 v0, v0
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s2, v0
@@ -253,13 +253,13 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -271,10 +271,10 @@ loop_body:
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -287,14 +287,14 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
%sum_e2 = add i32 %sum_i, %lds_val_n_2
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -316,7 +316,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, 4
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v7, 4
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: s_add_co_i32 s7, s10, 4
@@ -325,17 +325,17 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0
; SDAG-NEXT: s_load_b32 s12, s[0:1], 0x0
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
+; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v8, v0, s[8:9] offset:4 nv
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: s_clause 0x1
-; SDAG-NEXT: global_load_b32 v2, v0, s[8:9] offset:4
-; SDAG-NEXT: global_load_b32 v3, v0, s[0:1] offset:4
-; SDAG-NEXT: global_load_async_to_lds_b32 v9, v5, s[8:9] offset:4 nv
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
+; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
+; SDAG-NEXT: global_load_async_to_lds_b32 v9, v7, s[8:9] offset:4 nv
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_mov_b32 s8, 2
; SDAG-NEXT: s_mov_b32 s9, s10
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s12
; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
@@ -346,17 +346,18 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v9, s12
-; SDAG-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v10, s9
-; SDAG-NEXT: global_load_b32 v1, v0, s[4:5]
-; SDAG-NEXT: v_dual_add_nc_u32 v11, v6, v7 :: v_dual_mov_b32 v7, v3
+; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
+; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v10, s9
+; SDAG-NEXT: global_load_b32 v3, v0, s[4:5]
+; SDAG-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_add_nc_u32 v11, v5, v6
+; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: global_load_b32 v4, v0, s[0:1]
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: ds_load_b32 v10, v10
-; SDAG-NEXT: v_mov_b32_e32 v6, v2
+; SDAG-NEXT: v_mov_b32_e32 v5, v1
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
; SDAG-NEXT: s_cmp_lt_i32 s8, s11
@@ -372,7 +373,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_add_co_i32 s0, s11, -2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10
; SDAG-NEXT: s_wait_loadcnt 0x0
@@ -381,15 +382,15 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: ds_load_b32 v2, v0
; SDAG-NEXT: v_mov_b32_e32 v5, s0
; SDAG-NEXT: s_add_co_i32 s0, s11, -1
-; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u32_e32 v2, v3, v2
-; SDAG-NEXT: global_store_b32 v5, v2, s[2:3] scale_offset
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_dual_add_nc_u32 v1, v2, v1 :: v_dual_add_nc_u32 v2, v3, v4
+; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_add_nc_u32 v0, v1, v0
-; SDAG-NEXT: global_store_b32 v1, v0, s[2:3] scale_offset
+; SDAG-NEXT: v_dual_add_nc_u32 v0, v2, v0 :: v_dual_mov_b32 v2, s0
+; SDAG-NEXT: global_store_b32 v2, v0, s[2:3] scale_offset
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_pipelined_loop_with_global:
@@ -403,18 +404,18 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
-; GISEL-NEXT: s_mov_b32 s13, 2
+; GISEL-NEXT: s_mov_b32 s12, 2
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_load_b32 s12, s[8:9], 0x0
+; GISEL-NEXT: s_load_b32 s13, s[8:9], 0x0
; GISEL-NEXT: s_load_b32 s16, s[0:1], 0x0
; GISEL-NEXT: v_mov_b32_e32 v1, s10
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v5, s6
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
+; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_b32 v2, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v3, v0, s[0:1] offset:4
; GISEL-NEXT: global_load_async_to_lds_b32 v5, v4, s[8:9] offset:4 nv
@@ -424,30 +425,31 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_mov_b32 s14, s10
; GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s15, v3
; GISEL-NEXT: s_mov_b32 s9, s8
-; GISEL-NEXT: s_mov_b32 s17, s15
+; GISEL-NEXT: s_mov_b32 s19, s15
; GISEL-NEXT: .LBB2_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: s_add_co_u32 s18, s14, 8
+; GISEL-NEXT: s_add_co_u32 s17, s14, 8
; GISEL-NEXT: global_load_b32 v2, v0, s[6:7]
-; GISEL-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v4, s14
-; GISEL-NEXT: s_mov_b32 s19, s9
+; GISEL-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v4, s14
+; GISEL-NEXT: s_mov_b32 s18, s9
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_add_co_i32 s9, s12, s16
-; GISEL-NEXT: s_add_co_i32 s13, s13, 1
+; GISEL-NEXT: s_add_co_i32 s9, s13, s16
+; GISEL-NEXT: s_add_co_i32 s12, s12, 1
+; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[6:7] offset:4 nv
; GISEL-NEXT: global_load_b32 v3, v0, s[0:1]
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v4, v4
-; GISEL-NEXT: s_mov_b32 s18, s17
-; GISEL-NEXT: s_mov_b32 s12, s8
+; GISEL-NEXT: s_mov_b32 s17, s19
+; GISEL-NEXT: s_mov_b32 s13, s8
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s16, v4
; GISEL-NEXT: s_add_co_i32 s9, s9, s16
@@ -464,18 +466,18 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
; GISEL-NEXT: s_add_co_u32 s14, s14, 4
-; GISEL-NEXT: s_cmp_lt_i32 s13, s11
+; GISEL-NEXT: s_cmp_lt_i32 s12, s11
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s17, v3
+; GISEL-NEXT: v_readfirstlane_b32 s19, v3
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
-; GISEL-NEXT: s_add_co_i32 s4, s19, s18
+; GISEL-NEXT: s_add_co_i32 s4, s18, s17
; GISEL-NEXT: s_add_co_u32 s1, s10, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s1
@@ -486,9 +488,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_i32 s1, s4, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v2, s1
-; GISEL-NEXT: s_add_co_i32 s1, s9, s17
+; GISEL-NEXT: s_add_co_i32 s1, s9, s19
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0 offset:4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -504,7 +506,7 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
@@ -514,7 +516,7 @@ prolog:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -532,10 +534,10 @@ loop_body:
%cur_g = load i32, ptr addrspace(1) %bar_gep_cur
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -551,7 +553,7 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
@@ -560,7 +562,7 @@ epilog:
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
store i32 %sum_e2, ptr addrspace(1) %out_gep_e1
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -582,9 +584,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v3, v2
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -597,9 +599,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v3, v2
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -607,9 +609,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -628,11 +630,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_add_nc_u32_e32 v3, 4, v2
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v6, v2
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -647,11 +649,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v2
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v4, v2
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -660,11 +662,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-loop-carried.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-loop-carried.mir
new file mode 100644
index 00000000000000..074b7074e19525
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-loop-carried.mir
@@ -0,0 +1,227 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# The only asyncmark that reaches each wait.asyncmark below comes around the
+# loop backedge, so the first visit of the loop body sees an empty sequence.
+# The wait must still be generated once the backedge state is merged in, like
+# the loop-carried soft waitcnts of https://github.com/llvm/llvm-project/pull/220357.
+# The wait.asyncmark is never erased, so every visit recomputes its wait from
+# the merged state.
+
+---
+name: loop_carried_mark
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_carried_mark
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ WAIT_ASYNCMARK 0, 2
+ $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
+
+---
+name: loop_carried_mark_with_soft_waits
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_carried_mark_with_soft_waits
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: S_BARRIER
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_WAIT_DSCNT_soft 0
+ S_BARRIER
+ S_WAIT_DSCNT_soft 0
+ WAIT_ASYNCMARK 0, 2
+ $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
+
+---
+name: loop_carried_mark_other_stage
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_carried_mark_other_stage
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_STORE_ASYNC_FROM_LDS
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ WAIT_ASYNCMARK 0, 2
+ $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1)
+ ASYNCMARK 8
+ S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
+
+# A soft wait directly before the wait.asyncmark looks redundant on the first
+# visit and is queued for erasure, while the wait.asyncmark finds no marks yet.
+# On the revisit both are needed: dscnt for the loop-carried DS write, and
+# asynccnt for the loop-carried mark.
+---
+name: loop_carried_mark_adjacent_soft_wait
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_carried_mark_adjacent_soft_wait
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
+
+ S_WAIT_DSCNT_soft 0
+ WAIT_ASYNCMARK 0, 2
+ $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
index 79017c052d0cbb..cec31858c77032 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck %s
@@ -10,26 +11,26 @@
; Loop body: 18 markers
; CHECK-LABEL: test_loop_exceeds_max_first_iteration:
-; CHECK: ; wait_asyncmark(3)
+; CHECK: ; wait_asyncmark(3, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_exceeds_max_first_iteration(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
@@ -42,49 +43,49 @@ loop_header:
loop_body:
; Loop body with 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -95,26 +96,26 @@ exit:
; Loop body: 5 markers
; CHECK-LABEL: test_loop_needs_more_iterations:
-; CHECK: ; wait_asyncmark(3)
+; CHECK: ; wait_asyncmark(3, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_needs_more_iterations(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
@@ -127,20 +128,20 @@ loop_header:
loop_body:
; Loop body with 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -149,7 +150,7 @@ exit:
; Merge exceeds MaxAsyncMarkers
; CHECK-LABEL: max_when_merged:
-; CHECK: ; wait_asyncmark(17)
+; CHECK: ; wait_asyncmark(17, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(15)
define void @max_when_merged(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
@@ -160,68 +161,68 @@ entry:
then:
; 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %endif
else:
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %endif
endif:
- call void @llvm.amdgcn.wait.asyncmark(i16 17)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -230,53 +231,55 @@ endif:
; Straightline exceeds MaxAsyncMarkers
; CHECK-LABEL: no_max_in_straightline:
-; CHECK: ; wait_asyncmark(17)
+; CHECK: ; wait_asyncmark(17, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(17)
define void @no_max_in_straightline(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 17)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
index 3282ba05896fc3..d064180642f375 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
@@ -43,7 +43,7 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -52,7 +52,7 @@ body: |
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -61,27 +61,27 @@ body: |
S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
S_CBRANCH_SCC1 %bb.2, implicit killed $scc
- ; then branch — issues async LDS DMA + ASYNCMARK
+ ; then branch - issues async LDS DMA + ASYNCMARK
bb.1:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 2047
S_BRANCH %bb.3
- ; else branch — sync path, no async operations; OtherMarks is empty at join
+ ; else branch - sync path, no async operations; OtherMarks is empty at join
bb.2:
S_BRANCH %bb.3
- ; join — mergeAsyncMarks sees non-empty AsyncMarks (then) and empty OtherMarks (else).
+ ; join - mergeAsyncMarks sees non-empty AsyncMarks (then) and empty OtherMarks (else).
; Before fix: assertion. After fix: returns early, no spurious wait inserted.
bb.3:
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 2047
S_ENDPGM 0
...
---
# Pattern 2: ASYNCMARK in else-successor, then-successor is sync.
-# Mirror of asyncmark_in_then — exercises the opposite predecessor ordering.
+# Mirror of asyncmark_in_then - exercises the opposite predecessor ordering.
name: asyncmark_in_else
tracksRegLiveness: true
machineFunctionInfo:
@@ -107,11 +107,11 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -120,20 +120,20 @@ body: |
S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
S_CBRANCH_SCC1 %bb.2, implicit killed $scc
- ; then branch — sync path, no async operations
+ ; then branch - sync path, no async operations
bb.1:
S_BRANCH %bb.3
- ; else branch — issues async LDS DMA + ASYNCMARK
+ ; else branch - issues async LDS DMA + ASYNCMARK
bb.2:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 2047
S_BRANCH %bb.3
; join block
bb.3:
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 2047
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
index dd41980f42b14e..23a5e296ce8638 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
@@ -39,13 +39,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $m0 = S_MOV_B32 0
; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_ENDPGM 0
@@ -75,13 +75,13 @@ body: |
$m0 = S_MOV_B32 0
GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 2047
S_BRANCH %bb.3
bb.3:
liveins: $vgpr2
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 2047
renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
S_ENDPGM 0
...
@@ -107,7 +107,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $m0 = S_MOV_B32 0
; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -124,7 +124,7 @@ body: |
; CHECK-NEXT: bb.3:
; CHECK-NEXT: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_ENDPGM 0
@@ -142,7 +142,7 @@ body: |
$m0 = S_MOV_B32 0
GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 2047
S_BRANCH %bb.3
; branch target: load predecessor, merged first (seeds incoming state)
@@ -160,7 +160,7 @@ body: |
bb.3:
liveins: $vgpr2
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 2047
renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 9d5b9e5b2c7d4a..26a59bfa0dba2e 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -13,7 +13,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: ds_read_b32 v1, v2 offset:4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v0, v1
@@ -25,7 +25,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: ds_read_b32 v1, v2 offset:4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v0, v1
@@ -34,7 +34,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL-NEXT: s_setpc_b64 s[30:31]
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%val1 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val2 = load i32, ptr addrspace(3) %lds_gep1
%sum = add i32 %val1, %val2
store i32 %sum, ptr addrspace(3) %out
@@ -58,18 +58,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[3:4], off
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v4, v8, v7
@@ -91,18 +91,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[3:4], off
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v4, v8, v7
@@ -119,7 +119,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -127,16 +127,16 @@ entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma
; This results in vmcnt(1), corresponding to %bar_v12. Could have been combined with the lgkmcnt(1) for %lds_val21.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -160,7 +160,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: v_mov_b32_e32 v8, 0x54
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -168,11 +168,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -193,7 +193,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: v_mov_b32_e32 v8, 0x54
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -201,11 +201,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -222,7 +222,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -230,16 +230,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -264,7 +264,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -272,10 +272,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -297,7 +297,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -305,10 +305,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -326,7 +326,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
fence release
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -334,16 +334,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -368,20 +368,20 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v5, 0
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: .LBB4_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s7, v2
; SDAG-NEXT: s_mov_b32 m0, s7
; SDAG-NEXT: s_add_i32 s6, s6, 1
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: ds_read_b32 v6, v2
; SDAG-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -392,10 +392,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_cbranch_execnz .LBB4_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v5, v0
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -409,21 +409,21 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 s7, 0
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v5, s7
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: .LBB4_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: v_readfirstlane_b32 s7, v2
; GISEL-NEXT: s_mov_b32 m0, s7
; GISEL-NEXT: s_add_i32 s6, s6, 1
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: ds_read_b32 v6, v2
; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -434,10 +434,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v5, v0
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -446,11 +446,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -460,10 +460,10 @@ loop_body:
; Load next iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -475,11 +475,11 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e2 = add i32 %sum_i, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e1 = add i32 %sum_e2, %lds_val_n_1
store i32 %sum_e2, ptr addrspace(1) %bar
@@ -500,12 +500,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: global_load_dword v10, v[3:4], off
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: global_load_dword v12, v[0:1], off
-; SDAG-NEXT: global_load_dword v15, v[3:4], off
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: global_load_dword v8, v[0:1], off
+; SDAG-NEXT: global_load_dword v9, v[3:4], off
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_mov_b32_e32 v9, v12
; SDAG-NEXT: s_waitcnt vmcnt(1)
@@ -527,17 +527,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: v_mov_b32_e32 v8, v12
; SDAG-NEXT: v_mov_b32_e32 v10, v15
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_cbranch_execnz .LBB5_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v1, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v3, v17, v16
@@ -561,12 +561,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: global_load_dword v10, v[3:4], off
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: global_load_dword v12, v[0:1], off
-; GISEL-NEXT: global_load_dword v15, v[3:4], off
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: global_load_dword v8, v[0:1], off
+; GISEL-NEXT: global_load_dword v9, v[3:4], off
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: v_mov_b32_e32 v9, v12
; GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -588,17 +588,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: v_mov_b32_e32 v8, v12
; GISEL-NEXT: v_mov_b32_e32 v10, v15
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v1, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v3, v17, v16
@@ -617,13 +617,13 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%v1 = load i32, ptr addrspace(1) %foo
%g1 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -646,10 +646,10 @@ loop_body:
%cur_v = load i32, ptr addrspace(1) %foo
%cur_g = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -662,13 +662,13 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e0 = add i32 %sum, %g1_phi
%sum_e1 = add i32 %v1_phi, %sum_e0
%sum_e2 = add i32 %sum_e1, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e3 = add i32 %cur_v, %cur_g
%sum_e4 = add i32 %sum_e3, %lds_val_n_1
@@ -689,9 +689,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -706,9 +706,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -717,9 +717,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -739,12 +739,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_readfirstlane_b32 s4, v5
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: s_mov_b32 m0, s4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -761,12 +761,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_readfirstlane_b32 s4, v5
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -776,11 +776,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
new file mode 100644
index 00000000000000..b6e065c503546d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
@@ -0,0 +1,30 @@
+; RUN: split-file %s %t
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/mark.ll 2>&1 | FileCheck %s -check-prefix=MARK
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/wait.ll 2>&1 | FileCheck %s -check-prefix=WAIT
+
+; The stage mask operand carries a Range covering the bits of the defined
+; stages, so the IR verifier rejects a mask naming a stage that does not exist
+; before instruction selection. Bits of reserved stages are inside the range and
+; so are accepted: naming a stage whose operations do not exist yet is harmless,
+; and keeps masks portable as the reserved slots are filled in.
+
+;--- mark.ll
+; 2048 is the bit just past the last stage, and 4096 is further out still.
+; MARK: immarg value 2048 for arg 0 out of range [0,2048)
+; MARK: immarg value 4096 for arg 0 out of range [0,2048)
+define amdgpu_kernel void @asyncmark_mask_out_of_range() {
+ call void @llvm.amdgcn.asyncmark(i32 2048)
+ call void @llvm.amdgcn.asyncmark(i32 4096)
+ ret void
+}
+
+;--- wait.ll
+; A mask made only of reserved stage bits is fine, so the sole error here is the
+; out-of-range one.
+; WAIT: immarg value 2048 for arg 1 out of range [0,2048)
+; WAIT-NOT: out of range
+define amdgpu_kernel void @wait_asyncmark_mask_out_of_range() {
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2000)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2048)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
new file mode 100644
index 00000000000000..99c8fbf7dc7f5f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
@@ -0,0 +1,219 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# The MaxAsyncMarks cap (16) is applied per stage, since each stage owns its own
+# mark sequence. Saturating one stage must not perturb another.
+
+---
+# Mask 2 covers only GLOBAL_LOAD_ASYNC_TO_LDS, which receives 20 marks, well
+# past the cap. Mask 1 covers only TENSOR, which receives exactly one. The
+# wait covering TENSOR must still find its single mark and produce an exact
+# tensorcnt wait: it is not affected by the other stage's truncation.
+name: saturate_one_stage_only
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ ; CHECK-LABEL: name: saturate_one_stage_only
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK .TENSOR
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR
+ ; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+
+ ; TENSOR has one mark; counting back 0 reaches it exactly.
+ WAIT_ASYNCMARK 0, 1
+ S_ENDPGM 0
+...
+---
+# Waiting deep into a saturated stage. N is clamped to MaxAsyncMarks-1 so that a
+# non-trivial wait is still generated after truncation.
+name: wait_deep_in_saturated_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ ; CHECK-LABEL: name: wait_deep_in_saturated_stage
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: WAIT_ASYNCMARK 18, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 18, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+
+ WAIT_ASYNCMARK 18, 2
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
new file mode 100644
index 00000000000000..7dfb357e53d710
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
@@ -0,0 +1,285 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# Merging asyncmark brackets at a CFG join, with marks in different stages.
+#
+# Each stage keeps an independent mark sequence, so the join must merge each
+# sequence against its counterpart in the other predecessor and no other. The
+# zero-mark identity front-padding that equalises differing sequence lengths
+# has to be applied per stage rather than once across all of them.
+
+---
+# Both predecessors mark, but in different stages. Neither stage's sequence has
+# a counterpart in the other predecessor, so both survive the join as
+# single-element sequences padded with the zero-mark identity.
+#
+# The join then waits with a mask covering GLOBAL_LOAD_ASYNC_TO_LDS only. The
+# tensor DMA from bb.2 must stay in flight: no S_WAIT_TENSORCNT may appear.
+name: merge_disjoint_stages
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_disjoint_stages
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK .TENSOR
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Mask 2045 covers GLOBAL_LOAD_ASYNC_TO_LDS only, which is on ASYNC_CNT.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_BRANCH %bb.3
+
+ ; Mask 2046 covers TENSOR only, which is on TENSOR_CNT.
+ bb.2:
+ liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 2
+ S_ENDPGM 0
+...
+---
+# One predecessor is empty in the stage being waited on. Merging a one-element
+# sequence against an empty one must not disturb the other stage's sequence,
+# which is present in both predecessors.
+name: merge_one_side_empty_in_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_one_side_empty_in_stage
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK .TENSOR
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK .TENSOR
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .TENSOR
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Marks in both TENSOR and GLOBAL_LOAD_ASYNC_TO_LDS.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_BRANCH %bb.3
+
+ ; Marks in TENSOR only; GLOBAL_LOAD_ASYNC_TO_LDS's sequence is empty here.
+ bb.2:
+ liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 1
+ WAIT_ASYNCMARK 0, 2
+ S_ENDPGM 0
+...
+---
+# Sequences of differing length in the same stage. The shorter one is padded at
+# the front with zero marks, so counting back N marks from the join reaches the
+# conservative (most recent) mark on both paths.
+name: merge_unequal_lengths_same_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_unequal_lengths_same_stage
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Two marks in GLOBAL_LOAD_ASYNC_TO_LDS.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_BRANCH %bb.3
+
+ ; One mark in GLOBAL_LOAD_ASYNC_TO_LDS.
+ bb.2:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 2
+ S_ENDPGM 0
+...
+---
+# A wide mark and a narrow one merged at the same join. bb.1's zero mask joins
+# every stage, so it lands in GLOBAL_LOAD_ASYNC_TO_LDS's sequence alongside
+# bb.2's narrow mark, and those two merge against each other. bb.1's copies in
+# the other ten stages have no counterpart in bb.2 and are padded instead.
+name: merge_wide_and_narrow_marks
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_wide_and_narrow_marks
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .AllStages
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .AllStages
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 0
+ S_BRANCH %bb.3
+
+ bb.2:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 2
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 0
+ WAIT_ASYNCMARK 0, 2
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
new file mode 100644
index 00000000000000..380859e67413ae
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=GISEL
+
+; Stage BUFFER_GLOBAL_LOAD maps to LOAD_CNT, which pre-gfx12 is vmcnt. Buffer
+; async LDS loads are the operations in that stage.
+
+; Marks covering that stage form their own sequence; a wait covering it counts
+; back through them and produces a vmcnt wait, not a blanket wait.
+
+define amdgpu_kernel void @stage_buffer_global_load(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: stage_buffer_global_load:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_mov_b32 m0, s6
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; SDAG-NEXT: ; asyncmark(stages=BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
+; SDAG-NEXT: ; asyncmark(stages=BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
+; SDAG-NEXT: v_mov_b32_e32 v0, s6
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; SDAG-NEXT: ; wait_asyncmark(1, stages=BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: s_waitcnt vmcnt(2)
+; SDAG-NEXT: ds_read_b32 v0, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v1, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: stage_buffer_global_load:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_mov_b32 m0, s6
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; GISEL-NEXT: ; asyncmark(stages=BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
+; GISEL-NEXT: ; asyncmark(stages=BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
+; GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; GISEL-NEXT: ; wait_asyncmark(1, stages=BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: s_waitcnt vmcnt(2)
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 32)
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 32)
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 32)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; A wait covering a stage that never received a mark is a no-op, even though the
+; outstanding buffer loads are on the counter that stage would map to.
+
+define amdgpu_kernel void @wait_on_empty_stage(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: wait_on_empty_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_mov_b32 m0, s6
+; SDAG-NEXT: v_mov_b32_e32 v0, s6
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ds_read_b32 v0, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v1, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_on_empty_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_mov_b32 m0, s6
+; GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 32)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
+declare void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32>, ptr addrspace(3), i32, i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
new file mode 100644
index 00000000000000..da7603c2407af8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
@@ -0,0 +1,784 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -verify-machineinstrs -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GISEL
+
+; Each async stage has its own sequence of marks. The intrinsics take a mask of
+; the stages to act on: asyncmark marks them, wait_asyncmark waits on them. So
+; 1 covers TENSOR alone, and 2 covers GLOBAL_LOAD_ASYNC_TO_LDS alone. A zero
+; mask is special and covers every stage.
+
+; The over-waiting fix. Tensor DMA (TENSOR_CNT) and async global-to-LDS
+; (ASYNC_CNT) are interleaved, each marked into its own stage. A wait covering
+; TENSOR alone must wait only on tensorcnt: the outstanding async load is
+; unrelated to it and must stay in flight.
+
+define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_tensor_ignores_async:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v2, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v2, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_tensor_ignores_async:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v2, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v2, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; Waits for the tensor DMA only. No s_wait_asynccnt may be emitted here.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; Same shape, but covering the async stage instead. Now asynccnt is required
+; and tensorcnt is not.
+
+define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_async_ignores_tensor:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: ds_load_b32 v2, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v2, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_async_ignores_tensor:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: ds_load_b32 v2, v1
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v2, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; Two stages that share ASYNC_CNT. Stages are still independent mark sequences:
+; a wait covering GLOBAL_LOAD_ASYNC_TO_LDS counts back through the loads only,
+; so the intervening store mark does not shift the wait. Both loads and the store
+; are on asynccnt, so a wait is emitted, but its value reflects one sequence.
+
+define amdgpu_kernel void @same_counter_distinct_sequences(ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: same_counter_distinct_sequences:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_add_co_i32 s3, s2, 4
+; SDAG-NEXT: s_add_co_i32 s2, s2, 8
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x2
+; SDAG-NEXT: ds_load_b32 v4, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v4, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: same_counter_distinct_sequences:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GISEL-NEXT: s_add_co_u32 s3, s2, 4
+; GISEL-NEXT: s_add_co_u32 s2, s2, 8
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x2
+; GISEL-NEXT: ds_load_b32 v4, v1
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v4, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ %lds2 = getelementptr i32, ptr addrspace(3) %lds, i32 2
+
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark #0.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; GLOBAL_STORE_ASYNC_FROM_LDS mark #0. Same counter, different sequence.
+ call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 8)
+
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark #1.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds2, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; Counts back one mark in the load stage, skipping the store stage's mark.
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A zero mask names every stage, so the mark joins every stage and the wait
+; covers every stage. This is the behavior of the original stage-less
+; intrinsics, and is what old IR auto-upgrades to.
+
+define amdgpu_kernel void @zero_mask_covers_everything(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: zero_mask_covers_everything:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v2, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v2, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: zero_mask_covers_everything:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v2, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v2, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A wait covering a stage with no marks at all is a no-op: nothing to count back
+; to.
+
+define amdgpu_kernel void @wait_on_empty_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_on_empty_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; SDAG-NEXT: ds_load_b32 v2, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v2, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_on_empty_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; GISEL-NEXT: ds_load_b32 v2, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v2, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; TENSOR never had a mark, so this waits for nothing.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; The multicast (cluster) loads have their own stage, distinct from the plain
+; async loads to LDS even though both use the async counter. A wait covering the
+; multicast stage must not count back through a plain load's mark.
+define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 %mask) {
+; SDAG-LABEL: mcast_is_a_distinct_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_add_co_i32 s2, s2, 4
+; SDAG-NEXT: s_mov_b32 m0, s3
+; SDAG-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: ds_load_b32 v3, v1 offset:4
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v3, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: mcast_is_a_distinct_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GISEL-NEXT: s_add_co_u32 s2, s2, 4
+; GISEL-NEXT: s_mov_b32 m0, s3
+; GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: ds_load_b32 v3, v1 offset:4
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v3, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; GLOBAL_LOAD_ASYNC_TO_LDS_MCAST mark. Same counter, own sequence.
+ call void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0, i32 %mask)
+ call void @llvm.amdgcn.asyncmark(i32 4)
+
+ ; Drains the multicast stage only. The plain load's mark is in another
+ ; sequence.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 4)
+ %v = load i32, ptr addrspace(3) %lds1
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A mark whose mask names several stages joins all of them at once, and each
+; stage keeps its own copy. Here one mark covers both TENSOR and
+; GLOBAL_LOAD_ASYNC_TO_LDS, so a wait on either one alone finds it.
+
+define amdgpu_kernel void @one_mark_joins_two_stages(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: one_mark_joins_two_stages:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=TENSOR|GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v2, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v2, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: one_mark_joins_two_stages:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=TENSOR|GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v2, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v2, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+
+ ; Joins TENSOR and GLOBAL_LOAD_ASYNC_TO_LDS.
+ call void @llvm.amdgcn.asyncmark(i32 3)
+
+ ; Consumes the TENSOR copy. The GLOBAL_LOAD_ASYNC_TO_LDS copy is untouched, so
+ ; only tensorcnt is waited on.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A wait covering several stages is the union of their requirements. The two
+; stages here sit on different counters, so both a tensorcnt and an asynccnt
+; wait are emitted, each derived from its own sequence.
+
+define amdgpu_kernel void @multi_stage_wait_unions_counters(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: multi_stage_wait_unions_counters:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: s_add_co_i32 s4, s18, 4
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_store_async_from_lds_b32 v0, v1, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR|GLOBAL_STORE_ASYNC_FROM_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x1
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v3, v2
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v3, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: multi_stage_wait_unions_counters:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: s_add_co_u32 s4, s18, 4
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GISEL-NEXT: v_mov_b32_e32 v2, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_store_async_from_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v0, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR|GLOBAL_STORE_ASYNC_FROM_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x1
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v3, v2
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v3, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 8)
+
+ ; This async load is marked into neither stage, so neither wait covers it.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; Covers TENSOR and GLOBAL_STORE_ASYNC_FROM_LDS together.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 9)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A software pipeline that waits on a stage at the top of the loop and marks it
+; at the bottom, so the marks reaching the wait come both from the preheader and
+; around the backedge. In steady state GLOBAL_LOAD_ASYNC_TO_LDS holds two marks
+; at the wait, so waiting down to one leaves the most recent load in flight. A
+; tensor load marked in its own stage stays in flight across the whole loop: the
+; wait in the loop must never wait on tensorcnt, and the tensor mark survives
+; the loop to be waited on at the exit.
+
+define amdgpu_kernel void @loop_wait_then_mark_same_stage(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 %n) {
+; SDAG-LABEL: loop_wait_then_mark_same_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x64 nv
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: tensor_load_to_lds s[16:19], s[8:15]
+; SDAG-NEXT: s_add_co_i32 s4, s2, 4
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s4
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: s_mov_b32 s4, 0
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:8
+; SDAG-NEXT: s_mov_b32 s5, s2
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: .LBB8_1: ; %loop
+; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: s_add_co_i32 s6, s5, 8
+; SDAG-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x1
+; SDAG-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s5
+; SDAG-NEXT: s_add_co_i32 s4, s4, 1
+; SDAG-NEXT: s_add_co_i32 s5, s5, 4
+; SDAG-NEXT: s_cmp_lt_u32 s4, s3
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:12
+; SDAG-NEXT: ds_load_b32 v3, v3
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v3
+; SDAG-NEXT: s_cbranch_scc1 .LBB8_1
+; SDAG-NEXT: ; %bb.2: ; %exit
+; SDAG-NEXT: s_add_co_i32 s2, s2, -4
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-NEXT: ds_load_b32 v0, v0
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v1, v0
+; SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: loop_wait_then_mark_same_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x64 nv
+; GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_mov_b32 s5, 0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: tensor_load_to_lds s[16:19], s[8:15]
+; GISEL-NEXT: s_add_co_u32 s4, s2, 4
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s4
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: s_mov_b32 s6, s2
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:8
+; GISEL-NEXT: s_mov_b32 s4, 0
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: .LBB8_1: ; %loop
+; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: s_add_co_u32 s7, s6, 8
+; GISEL-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x1
+; GISEL-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s6
+; GISEL-NEXT: s_add_co_i32 s5, s5, 1
+; GISEL-NEXT: s_add_co_u32 s6, s6, 4
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:12
+; GISEL-NEXT: ds_load_b32 v2, v2
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: v_readfirstlane_b32 s7, v2
+; GISEL-NEXT: s_add_co_i32 s4, s4, s7
+; GISEL-NEXT: s_cmp_lt_u32 s5, s3
+; GISEL-NEXT: s_cbranch_scc1 .LBB8_1
+; GISEL-NEXT: ; %bb.2: ; %exit
+; GISEL-NEXT: s_add_co_u32 s2, s2, -4
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-NEXT: s_add_co_i32 s2, s4, s2
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 8, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2)
+ %lds.cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
+ %v = load i32, ptr addrspace(3) %lds.cur
+ %acc.next = add i32 %acc, %v
+ %i.ahead = add i32 %i, 2
+ %lds.ahead = getelementptr i32, ptr addrspace(3) %lds, i32 %i.ahead
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds.ahead, i32 12, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+ %i.next = add i32 %i, 1
+ %cmp = icmp ult i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ %lds.t = getelementptr i32, ptr addrspace(3) %lds, i32 -1
+ %t = load i32, ptr addrspace(3) %lds.t
+ %r = add i32 %acc.next, %t
+ store i32 %r, ptr addrspace(1) %glb
+ ret void
+}
+
+; The loop waits on one stage but marks another, and both stages share
+; ASYNC_CNT. The marks of GLOBAL_STORE_ASYNC_FROM_LDS accumulate around the
+; backedge but must not reach the wait on GLOBAL_LOAD_ASYNC_TO_LDS, whose only
+; mark comes from the preheader and is consumed on the first iteration.
+;
+; The loop header merges the preheader's mark with the backedge's empty
+; sequence, so the wait is kept on every iteration. The merge also aligns the
+; preheader's pending load with the most recent operation on the backedge, so
+; the wait is conservatively a full one, as for any counter merged at a loop
+; header.
+
+define amdgpu_kernel void @loop_wait_then_mark_other_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 %n) {
+; SDAG-LABEL: loop_wait_then_mark_other_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_mov_b32 s2, 0
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: .LBB9_1: ; %loop
+; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: global_store_async_from_lds_b32 v0, v1, s[0:1] offset:8
+; SDAG-NEXT: s_add_co_i32 s2, s2, 1
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: s_cmp_lt_u32 s2, s3
+; SDAG-NEXT: s_cbranch_scc1 .LBB9_1
+; SDAG-NEXT: ; %bb.2: ; %exit
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: loop_wait_then_mark_other_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GISEL-NEXT: s_mov_b32 s2, 0
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: .LBB9_1: ; %loop
+; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: global_store_async_from_lds_b32 v1, v0, s[0:1] offset:8
+; GISEL-NEXT: s_add_co_i32 s2, s2, 1
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: s_cmp_lt_u32 s2, s3
+; GISEL-NEXT: s_cbranch_scc1 .LBB9_1
+; GISEL-NEXT: ; %bb.2: ; %exit
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_STORE_ASYNC_FROM_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2)
+ call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 8, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 8)
+ %i.next = add i32 %i, 1
+ %cmp = icmp ult i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 8)
+ ret void
+}
+
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
+declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32>, <8 x i32>, <4 x i32>, <4 x i32>, <8 x i32>, i32)
+declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32)
+declare void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32)
+declare void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
index 683e2ba0af75b1..e7fa807c0d07de 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
@@ -17,9 +17,9 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK .TENSOR.GLOBAL_LOAD_ASYNC_TO_LDS.GLOBAL_LOAD_ASYNC_TO_LDS_MCAST.GLOBAL_STORE_ASYNC_FROM_LDS.BUFFER_GLOBAL_LOAD.RES7d0
; CHECK-NEXT: S_ENDPGM 0
S_WAITCNT_soft 0
- ASYNCMARK
+ ASYNCMARK 2047
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
index 681b584d3e48b3..c0ba4e1f957827 100644
--- a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
+++ b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
@@ -35,7 +35,7 @@ body: |
# WAIT_ASYNCMARK is a meta instruction that emits zero bytes.
# codeLenInByte = s_waitcnt (4) + WAIT_ASYNCMARK (0) = 4.
# CHECK: wait_asyncmark_meta: ; @wait_asyncmark_meta
-# CHECK: ; wait_asyncmark(1)
+# CHECK: ; wait_asyncmark(1, stages=all)
# CHECK: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
# CHECK: ; codeLenInByte = 4
---
@@ -44,7 +44,7 @@ tracksRegLiveness: true
body: |
bb.0:
- WAIT_ASYNCMARK 1
+ WAIT_ASYNCMARK 1, 0
...
# CHECK: align4: ; @align4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
index bfaf9239f7339a..9a60e908f6f5b0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -5,8 +5,8 @@
; %D4 should be zero-initialized for gfx1250, which only supports 4 groups of tensor descriptor
declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> %D4, i32 %cpol)
declare void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> %D4, i32 %cpol)
-declare void @llvm.amdgcn.asyncmark()
-declare void @llvm.amdgcn.wait.asyncmark(i16)
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 %offset, i32 %cpol)
define amdgpu_ps void @tensor_load_to_lds_d4(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
@@ -331,13 +331,13 @@ define amdgpu_ps void @tensor_load_to_lds_with_asyncmark(<4 x i32> inreg %D0, <8
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -349,13 +349,13 @@ define amdgpu_ps void @tensor_store_from_lds_with_asyncmark(<4 x i32> inreg %D0,
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -370,14 +370,14 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
+; GFX1250-NEXT: ; asyncmark(stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0xa
; GFX1250-NEXT: tensor_load_to_lds s[12:15], s[16:23]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(1)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x1
; GFX1250-NEXT: ds_load_b32 v1, v0
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: ds_load_b32 v2, v0 offset:4
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -385,15 +385,15 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: ds_store_b32 v0, v1
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0a, <8 x i32> %D1a, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0b, <8 x i32> %D1b, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_v0 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%lds_v1 = load i32, ptr addrspace(3) %lds_gep1
@@ -415,15 +415,15 @@ define void @tensor_and_async_lds_with_asyncmark(<4 x i32> inreg %D0, <8 x i32>
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[16:23]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_asynccnt 0x0
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -454,25 +454,25 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.1: ; %t1
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, 0
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-SDAG-NEXT: .LBB14_2: ; %t2
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0xa
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-SDAG-NEXT: ; asyncmark
-; GFX1250-SDAG-NEXT: ; wait_asyncmark(1)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
+; GFX1250-SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-SDAG-NEXT: s_wait_asynccnt 0x0
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0x1
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-SDAG-NEXT: .LBB14_3: ; %g1
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, 0
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB14_2
; GFX1250-SDAG-NEXT: .LBB14_4: ; %g2
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
-; GFX1250-SDAG-NEXT: ; asyncmark
-; GFX1250-SDAG-NEXT: ; wait_asyncmark(1)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
+; GFX1250-SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-SDAG-NEXT: s_wait_asynccnt 0x1
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -498,25 +498,25 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.1: ; %t1
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
; GFX1250-GISEL-NEXT: s_cmp_eq_u32 s1, 0
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-GISEL-NEXT: .LBB14_2: ; %t2
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0xa
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-GISEL-NEXT: ; asyncmark
-; GFX1250-GISEL-NEXT: ; wait_asyncmark(1)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
+; GFX1250-GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-GISEL-NEXT: s_wait_asynccnt 0x0
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0x1
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-NEXT: .LBB14_3: ; %g1
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_cmp_eq_u32 s1, 0
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB14_2
; GFX1250-GISEL-NEXT: .LBB14_4: ; %g2
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
-; GFX1250-GISEL-NEXT: ; asyncmark
-; GFX1250-GISEL-NEXT: ; wait_asyncmark(1)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
+; GFX1250-GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-GISEL-NEXT: s_wait_asynccnt 0x1
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0x0
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -526,12 +526,12 @@ entry:
t1:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge1
g1:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge1
merge1:
@@ -540,15 +540,15 @@ merge1:
t2:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge2
g2:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge2
merge2:
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
ret void
}
diff --git a/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir b/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
new file mode 100644
index 00000000000000..85d72a2c0d032f
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
@@ -0,0 +1,29 @@
+# RUN: split-file %s %t
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none %t/mark.mir -filetype=null 2>&1 | FileCheck %s -check-prefix=MARK
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none %t/wait.mir -filetype=null 2>&1 | FileCheck %s -check-prefix=WAIT
+
+#--- mark.mir
+# MARK: *** Bad machine code: invalid asyncmark stage mask ***
+# MARK-NEXT: - function: asyncmark_mask_out_of_range
+# MARK: ASYNCMARK 2048
+---
+name: asyncmark_mask_out_of_range
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ASYNCMARK 2048
+ S_ENDPGM 0
+...
+
+#--- wait.mir
+# WAIT: *** Bad machine code: invalid asyncmark stage mask ***
+# WAIT-NEXT: - function: wait_asyncmark_mask_out_of_range
+# WAIT: WAIT_ASYNCMARK 0, 4096
+---
+name: wait_asyncmark_mask_out_of_range
+tracksRegLiveness: true
+body: |
+ bb.0:
+ WAIT_ASYNCMARK 0, 4096
+ S_ENDPGM 0
+...
diff --git a/llvm/test/Transforms/DeadStoreElimination/AMDGPU/tensor-descriptor.ll b/llvm/test/Transforms/DeadStoreElimination/AMDGPU/tensor-descriptor.ll
index 2a39e5498e2460..3d0f2d3cddc774 100644
--- a/llvm/test/Transforms/DeadStoreElimination/AMDGPU/tensor-descriptor.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/AMDGPU/tensor-descriptor.ll
@@ -11,15 +11,15 @@ define void @tensor_load(<4 x i32> inreg %d0, <8 x i32> inreg %d1, ptr addrspace
; CHECK-SAME: <4 x i32> inreg [[D0:%.*]], <8 x i32> inreg [[D1:%.*]], ptr addrspace(1) [[GLOBAL:%.*]]) {
; CHECK-NEXT: store i32 1, ptr addrspace(1) [[GLOBAL]], align 4
; CHECK-NEXT: call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> [[D0]], <8 x i32> [[D1]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+; CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
; CHECK-NEXT: store i32 2, ptr addrspace(1) [[GLOBAL]], align 4
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(1) %global
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %d0, <8 x i32> %d1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
store i32 2, ptr addrspace(1) %global
ret void
}
@@ -30,15 +30,15 @@ define void @tensor_store(<4 x i32> inreg %d0, <8 x i32> inreg %d1, ptr addrspac
; CHECK-SAME: <4 x i32> inreg [[D0:%.*]], <8 x i32> inreg [[D1:%.*]], ptr addrspace(3) [[LDS:%.*]]) {
; CHECK-NEXT: store i32 1, ptr addrspace(3) [[LDS]], align 4
; CHECK-NEXT: call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> [[D0]], <8 x i32> [[D1]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+; CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
; CHECK-NEXT: store i32 2, ptr addrspace(3) [[LDS]], align 4
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(3) %lds
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %d0, <8 x i32> %d1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
store i32 2, ptr addrspace(3) %lds
ret void
}
diff --git a/llvm/test/Transforms/GVN/AMDGPU/tensor-descriptor.ll b/llvm/test/Transforms/GVN/AMDGPU/tensor-descriptor.ll
index 28acf87cd552e8..5e4214c21a5d0b 100644
--- a/llvm/test/Transforms/GVN/AMDGPU/tensor-descriptor.ll
+++ b/llvm/test/Transforms/GVN/AMDGPU/tensor-descriptor.ll
@@ -10,16 +10,16 @@ define i32 @tensor_load_writes(<4 x i32> inreg %d0, <8 x i32> inreg %d1, ptr add
; CHECK-SAME: <4 x i32> inreg [[D0:%.*]], <8 x i32> inreg [[D1:%.*]], ptr addrspace(3) [[DST:%.*]]) {
; CHECK-NEXT: [[BEFORE:%.*]] = load i32, ptr addrspace(3) [[DST]], align 4
; CHECK-NEXT: call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> [[D0]], <8 x i32> [[D1]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+; CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
; CHECK-NEXT: [[AFTER:%.*]] = load i32, ptr addrspace(3) [[DST]], align 4
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[BEFORE]], [[AFTER]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%before = load i32, ptr addrspace(3) %dst
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %d0, <8 x i32> %d1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%after = load i32, ptr addrspace(3) %dst
%sum = add i32 %before, %after
ret i32 %sum
@@ -31,16 +31,16 @@ define i32 @tensor_store_writes(<4 x i32> inreg %d0, <8 x i32> inreg %d1, ptr ad
; CHECK-SAME: <4 x i32> inreg [[D0:%.*]], <8 x i32> inreg [[D1:%.*]], ptr addrspace(1) [[DST:%.*]]) {
; CHECK-NEXT: [[BEFORE:%.*]] = load i32, ptr addrspace(1) [[DST]], align 4
; CHECK-NEXT: call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> [[D0]], <8 x i32> [[D1]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+; CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
; CHECK-NEXT: [[AFTER:%.*]] = load i32, ptr addrspace(1) [[DST]], align 4
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[BEFORE]], [[AFTER]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%before = load i32, ptr addrspace(1) %dst
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %d0, <8 x i32> %d1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%after = load i32, ptr addrspace(1) %dst
%sum = add i32 %before, %after
ret i32 %sum
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
index 075fa75ab5ad87..e1de0e861d6e76 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
@@ -85,6 +85,9 @@ def ROCDL_SchedGroupMaskAttr
: ROCDL_IntrinsicIntegerEnumAttr<ROCDL_SchedGroupMask,
"sched_group_mask">;
+def ROCDL_AsyncStagesAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_AsyncStages, "async_stages">;
+
def ROCDL_PreGfx12CachePolicyAttr
: ROCDL_IntrinsicIntegerEnumAttr<ROCDL_PreGfx12CachePolicy,
"pre_gfx12_cache_policy">;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
index fb704ecf4c0de8..e15db9e4f8dc1a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
@@ -161,6 +161,31 @@ def ROCDL_SchedGroupMask : ROCDL_I32BitEnum<"SchedGroupMask",
let printBitEnumQuoted = 0;
}
+// Mirrors llvm::AMDGPU::AsyncStages. The
+// zero mask names every stage rather than none.
+def ROCDL_AsyncStagesAll : I32BitEnumCaseNone<"all">;
+def ROCDL_AsyncStagesTensor : I32BitEnumCaseBit<"tensor", 0>;
+def ROCDL_AsyncStagesGlobalLoadAsyncToLDS
+ : I32BitEnumCaseBit<"global_load_async_to_lds", 1>;
+def ROCDL_AsyncStagesGlobalLoadAsyncToLDSMcast
+ : I32BitEnumCaseBit<"global_load_async_to_lds_mcast", 2>;
+def ROCDL_AsyncStagesGlobalStoreAsyncFromLDS
+ : I32BitEnumCaseBit<"global_store_async_from_lds", 3>;
+def ROCDL_AsyncStagesBufferGlobalLoad
+ : I32BitEnumCaseBit<"buffer_global_load", 5>;
+def ROCDL_AsyncStages : ROCDL_I32BitEnum<"AsyncStages",
+ "async stage mask for asyncmark and wait.asyncmark",
+ [
+ ROCDL_AsyncStagesAll,
+ ROCDL_AsyncStagesTensor,
+ ROCDL_AsyncStagesGlobalLoadAsyncToLDS,
+ ROCDL_AsyncStagesGlobalLoadAsyncToLDSMcast,
+ ROCDL_AsyncStagesGlobalStoreAsyncFromLDS,
+ ROCDL_AsyncStagesBufferGlobalLoad
+ ]> {
+ let printBitEnumQuoted = 0;
+}
+
def ROCDL_PreGfx12CachePolicyNone : I32BitEnumCaseNone<"none">;
def ROCDL_PreGfx12CachePolicyGLC : I32BitEnumCaseBit<"glc", 0>;
def ROCDL_PreGfx12CachePolicySLC : I32BitEnumCaseBit<"slc", 1>;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 87045c1cf54918..dc4ad56d70d792 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -675,8 +675,9 @@ def ROCDL_WaitTensorcntOp: ROCDL_ConcreteNonMemIntrOp<"s.wait.tensorcnt", [], 0,
let assemblyFormat = "$count attr-dict";
}
-def ROCDL_AsyncmarkOp : ROCDL_ConcreteNonMemIntrOp<"asyncmark", [], 0>,
- Arguments<(ins)> {
+def ROCDL_AsyncmarkOp : ROCDL_ConcreteNonMemIntrOp<"asyncmark", [], 0, [0], ["stages"]>,
+ Arguments<(ins DefaultValuedAttr<ROCDL_AsyncStagesAttr,
+ "::mlir::ROCDL::AsyncStages::all">:$stages)> {
let summary = "Mark the end of a group of asynchronous operations";
let description = [{
This operation, in conjunction with `rocdl.wait.asyncmark`, forms the
@@ -689,20 +690,27 @@ def ROCDL_AsyncmarkOp : ROCDL_ConcreteNonMemIntrOp<"asyncmark", [], 0>,
See `rocdl.wait.asyncmark`'s documentation for a usage example.
+ `stages` is a mask of the async stages to mark: the mark is appended to
+ the sequence of every stage it names. `all` names every stage.
+
Example:
```mlir
- // Mark the end of an async operation group.
+ // Mark the end of an async operation group in every stage.
rocdl.asyncmark
+ // Mark the end of a group of tensor loads and stores alone.
+ rocdl.asyncmark stages tensor
```
Available on gfx9 and later.
}];
let results = (outs);
- let assemblyFormat = "attr-dict";
+ let assemblyFormat = "(`stages` enum($stages)^)? attr-dict";
}
-def ROCDL_WaitAsyncmarkOp: ROCDL_ConcreteNonMemIntrOp<"wait.asyncmark", [], 0, [0], ["count"]>,
- Arguments<(ins I16Attr:$count)> {
+def ROCDL_WaitAsyncmarkOp: ROCDL_ConcreteNonMemIntrOp<"wait.asyncmark", [], 0, [0, 1], ["count", "stages"]>,
+ Arguments<(ins I16Attr:$count,
+ DefaultValuedAttr<ROCDL_AsyncStagesAttr,
+ "::mlir::ROCDL::AsyncStages::all">:$stages)> {
let summary = "Wait until N or fewer async operation groups are unexecuted";
let description = [{
This operation, along with `rocdl.asyncmark`, forms the compiler-provided
@@ -712,6 +720,10 @@ def ROCDL_WaitAsyncmarkOp: ROCDL_ConcreteNonMemIntrOp<"wait.asyncmark", [], 0, [
that were parts of any async group (established by asyncmark in program order)
other than the `count` previously-added ones will have finished executing.
+ `stages` is a mask of the async stages to wait on: each stage it names
+ is waited on separately, using its own sequence of asyncmarks. `all`
+ names every stage.
+
For more detail, including on how this mechanism composes with function calls,
see [the LLVM documentation on async tracking](/llvm/docs/AMDGPUAsyncOperations.md).
@@ -721,6 +733,8 @@ def ROCDL_WaitAsyncmarkOp: ROCDL_ConcreteNonMemIntrOp<"wait.asyncmark", [], 0, [
```mlir
// Wait until at most N async groups remain outstanding.
rocdl.wait.asyncmark 1
+ // The same, counting the tensor groups alone.
+ rocdl.wait.asyncmark 1 stages tensor
```
Usage example:
@@ -739,7 +753,7 @@ def ROCDL_WaitAsyncmarkOp: ROCDL_ConcreteNonMemIntrOp<"wait.asyncmark", [], 0, [
```
}];
let results = (outs);
- let assemblyFormat = "$count attr-dict";
+ let assemblyFormat = "$count (`stages` enum($stages)^)? attr-dict";
}
def ROCDL_SetPrioOp : ROCDL_ConcreteNonMemIntrOp<"s.setprio", [], 0, [0], ["priority"]>,
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 79c09dea8d301d..d3e1a23501ff73 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -2632,3 +2632,10 @@ llvm.func @nvvm_duplicate_fp_modifier(%a: f32, %b: f32) {
// expected-error at +1 {{failed to parse LoopAnnotationAttr parameter 'align' which is to be a `uint32_t`}}
#loop_align_too_large = #llvm.loop_annotation<align = 4294967296>
+// -----
+
+llvm.func @rocdl_asyncmark_unknown_stage() {
+ // expected-error at +1 {{custom op 'rocdl.asyncmark' expected string or keyword containing one of the following enum values for attribute 'stages' [all, tensor, global_load_async_to_lds, global_load_async_to_lds_mcast, global_store_async_from_lds, buffer_global_load]}}
+ rocdl.asyncmark stages not_a_stage
+ llvm.return
+}
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 1b5b6419368819..15ad14bbbb9176 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -1341,15 +1341,22 @@ llvm.func @rocdl.s.wait.tensorcnt() {
llvm.func @rocdl.asyncmark() {
// CHECK-LABEL: rocdl.asyncmark
- // CHECK: rocdl.asyncmark
+ // `all` is the default, so it is not printed.
+ // CHECK: rocdl.asyncmark{{$}}
+ // CHECK: rocdl.asyncmark{{$}}
+ // CHECK: rocdl.asyncmark stages tensor|global_load_async_to_lds_mcast{{$}}
rocdl.asyncmark
+ rocdl.asyncmark stages all
+ rocdl.asyncmark stages global_load_async_to_lds_mcast|tensor
llvm.return
}
llvm.func @rocdl.wait.asyncmark() {
// CHECK-LABEL: rocdl.wait.asyncmark
- // CHECK: rocdl.wait.asyncmark 0
+ // CHECK: rocdl.wait.asyncmark 0{{$}}
+ // CHECK: rocdl.wait.asyncmark 2 stages global_load_async_to_lds|buffer_global_load{{$}}
rocdl.wait.asyncmark 0
+ rocdl.wait.asyncmark 2 stages global_load_async_to_lds|buffer_global_load
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 2f067b8ba98fda..9b05261d8dc92c 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -399,15 +399,19 @@ llvm.func @rocdl.s.wait.tensorcnt() {
llvm.func @rocdl.asyncmark() {
// CHECK-LABEL: rocdl.asyncmark
- // CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
+ // CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+ // CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 5)
rocdl.asyncmark
+ rocdl.asyncmark stages tensor|global_load_async_to_lds_mcast
llvm.return
}
llvm.func @rocdl.wait.asyncmark() {
// CHECK-LABEL: rocdl.wait.asyncmark
- // CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ // CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ // CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 40)
rocdl.wait.asyncmark 0
+ rocdl.wait.asyncmark 2 stages global_store_async_from_lds|buffer_global_load
llvm.return
}
>From 3726500f8f80d83e4a5bd3962cabd2ec418b39e3 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Thu, 29 Oct 2026 08:27:01 -0700
Subject: [PATCH 2/3] Update asyncmark stages enum
---
clang/lib/Sema/SemaAMDGPU.cpp | 5 +-
llvm/include/llvm/Support/AMDGPUAsyncStages.h | 272 +++++++-----------
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp | 35 +--
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 47 +--
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +-
7 files changed, 152 insertions(+), 215 deletions(-)
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index 573dc662a9fbfa..0d2fd5f8803c53 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -234,8 +234,9 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(const TargetInfo &TI,
// stage bits is meaningful, including none of them: the zero mask names
// every stage.
unsigned MaskArgNum = IsMark ? 0 : 1;
- return SemaRef.BuiltinConstantArgRange(TheCall, MaskArgNum, 0,
- llvm::AMDGPU::AsyncStages::ALL);
+ return SemaRef.BuiltinConstantArgRange(
+ TheCall, MaskArgNum, 0,
+ llvm::to_underlying(llvm::AMDGPU::AsyncStages::ALL));
}
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
diff --git a/llvm/include/llvm/Support/AMDGPUAsyncStages.h b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
index e00ac8bc63e1bd..2cd62d49709488 100644
--- a/llvm/include/llvm/Support/AMDGPUAsyncStages.h
+++ b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
@@ -14,192 +14,124 @@
#ifndef LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
#define LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
+#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/bit.h"
-#include "llvm/ADT/iterator.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <cstdint>
-#include <iterator>
-namespace llvm {
-namespace AMDGPU {
+namespace llvm::AMDGPU {
+
+LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
/// Bit mask of the async stages tracked by the asyncmark / wait_asyncmark
-/// intrinsics. Each stage has its own independent sequence of marks.
-///
-/// A single stage is represented as a mask with exactly one bit set. Iterating
-/// over a mask visits each stage it names, from the least significant bit up.
+/// intrinsics. Each stage has its own independent sequence of marks. A single
+/// stage is a mask with exactly one bit set.
///
-/// This type also provides certain stronger guarantees than a simple integer:
-/// - Default constructor initializes the mask to zero.
-/// - Constructor ensures undefined bits cannot be set.
-class AsyncStages {
-public:
- using value_type = uint32_t;
-
- static constexpr unsigned NUM_STAGES = 11;
-
- // The bit of each stage is part of the IR, so do not renumber them. Some
- // values are RESERVED for later use.
- enum : value_type {
- NONE = 0,
- // Tensor loads to LDS and tensor stores from LDS.
- TENSOR = 1u << 0,
- // Asynchronous global loads to LDS.
- GLOBAL_LOAD_ASYNC_TO_LDS = 1u << 1,
- // Asynchronous multicast (cluster) global loads to LDS.
- GLOBAL_LOAD_ASYNC_TO_LDS_MCAST = 1u << 2,
- // Asynchronous global stores from LDS.
- GLOBAL_STORE_ASYNC_FROM_LDS = 1u << 3,
- // Held for downstream use.
- RESERVED_4 = 1u << 4,
- // Buffer loads to LDS and pre-gfx1250 global loads to LDS.
- BUFFER_GLOBAL_LOAD = 1u << 5,
- RESERVED_6 = 1u << 6,
- RESERVED_7 = 1u << 7,
- RESERVED_8 = 1u << 8,
- RESERVED_9 = 1u << 9,
- RESERVED_10 = 1u << 10,
-
- RESERVED = RESERVED_4 | RESERVED_6 | RESERVED_7 | RESERVED_8 | RESERVED_9 |
- RESERVED_10,
-
- // Bits that a mask may legally set. Reserved stages are included: naming a
- // stage whose operations do not exist yet is harmless, and accepting the
- // bit keeps masks portable as stages are filled in.
- ALL = (1u << NUM_STAGES) - 1
- };
-
- /// Iterates over the stages named by a mask.
- /// NOLINTNEXTLINE
- class const_iterator
- : public iterator_facade_base<const_iterator, std::forward_iterator_tag,
- AsyncStages> {
- // The "end" iterator is also the default-constructed iterator.
- // We naturally move towards the "end" by clearing the set bits from least
- // to most significant.
- AsyncStages::value_type Cur = 0;
-
- public:
- const_iterator() = default;
- const_iterator(AsyncStages S) : Cur(S.value()) {}
-
- bool operator==(const const_iterator &Other) const {
- return Cur == Other.Cur;
- }
-
- AsyncStages operator*() const {
- // Return only rightmost (least significant) bit set.
- return Cur ? (Cur & (1u << countr_zero(Cur))) : 0;
- }
-
- const_iterator &operator++() {
- // Keep all bits except the least significant bit set.
- Cur &= maskTrailingZeros<AsyncStages::value_type>(countr_zero(Cur) + 1);
- return *this;
- }
- };
-
- constexpr AsyncStages() = default;
- constexpr AsyncStages(value_type V) : Data(V) {
- assert((V & ALL) == V && "Bits set out of bounds!");
- }
-
- /// \returns true if \p Imm is a legal stage mask operand of the asyncmark /
- /// wait_asyncmark intrinsics and pseudos.
- static constexpr bool isValidMask(int64_t Imm) {
- return Imm >= 0 && (Imm & ~int64_t(ALL)) == 0;
- }
-
- /// \returns the stages named by the stage mask operand \p Imm of the
- /// asyncmark / wait_asyncmark intrinsics and pseudos. The zero mask is the
- /// one exception to a set bit naming a stage: it names every stage rather
- /// than none.
- static constexpr AsyncStages fromMaskOperand(value_type Imm) {
- return Imm ? AsyncStages(Imm) : AsyncStages(ALL);
- }
-
- constexpr unsigned size() const { return popcount(Data); }
- constexpr bool any() const { return Data != 0; }
- constexpr bool none() const { return Data == 0; }
- constexpr value_type value() const { return Data; }
-
- explicit constexpr operator bool() const { return any(); }
-
- const_iterator begin() const { return *this; }
- const_iterator end() const { return {}; }
-
- /// \returns the position of this single stage in a mask, for indexing
- /// per-stage data.
- unsigned index() const {
- assert(size() == 1 && "Expected a single stage");
- return countr_zero(Data);
- }
-
- /// \returns the name of this single stage.
- const char *getName() const {
- switch (Data) {
- case TENSOR:
- return "TENSOR";
- case GLOBAL_LOAD_ASYNC_TO_LDS:
- return "GLOBAL_LOAD_ASYNC_TO_LDS";
- case GLOBAL_LOAD_ASYNC_TO_LDS_MCAST:
- return "GLOBAL_LOAD_ASYNC_TO_LDS_MCAST";
- case GLOBAL_STORE_ASYNC_FROM_LDS:
- return "GLOBAL_STORE_ASYNC_FROM_LDS";
- case RESERVED_4:
- return "RESERVED_4";
- case BUFFER_GLOBAL_LOAD:
- return "BUFFER_GLOBAL_LOAD";
- case RESERVED_6:
- return "RESERVED_6";
- case RESERVED_7:
- return "RESERVED_7";
- case RESERVED_8:
- return "RESERVED_8";
- case RESERVED_9:
- return "RESERVED_9";
- case RESERVED_10:
- return "RESERVED_10";
- }
- llvm_unreachable("Expected a single stage");
- }
-
- constexpr AsyncStages operator&(AsyncStages Other) const {
- return Data & Other.Data;
- }
- constexpr AsyncStages operator-(AsyncStages Other) const {
- return Data & ~Other.Data;
- }
-
- constexpr bool operator==(AsyncStages Other) const {
- return Data == Other.Data;
- }
-
- constexpr AsyncStages &operator|=(AsyncStages Other) {
- Data |= Other.Data;
- return *this;
- }
-
-private:
- value_type Data = NONE;
+/// The bit of each stage is part of the IR, so do not renumber them. Some
+/// values are RESERVED for later use.
+enum class AsyncStages : uint32_t {
+ NONE = 0,
+ // Tensor loads to LDS and tensor stores from LDS.
+ TENSOR = 1u << 0,
+ // Asynchronous global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS = 1u << 1,
+ // Asynchronous multicast (cluster) global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS_MCAST = 1u << 2,
+ // Asynchronous global stores from LDS.
+ GLOBAL_STORE_ASYNC_FROM_LDS = 1u << 3,
+ // Held for downstream use.
+ RESERVED_4 = 1u << 4,
+ // Buffer loads to LDS and pre-gfx1250 global loads to LDS.
+ BUFFER_GLOBAL_LOAD = 1u << 5,
+ RESERVED_6 = 1u << 6,
+ RESERVED_7 = 1u << 7,
+ RESERVED_8 = 1u << 8,
+ RESERVED_9 = 1u << 9,
+ RESERVED_10 = 1u << 10,
+
+ RESERVED = RESERVED_4 | RESERVED_6 | RESERVED_7 | RESERVED_8 | RESERVED_9 |
+ RESERVED_10,
+
+ // Bits that a mask may legally set. Reserved stages are included: naming a
+ // stage whose operations do not exist yet is harmless, and accepting the bit
+ // keeps masks portable as stages are filled in.
+ ALL = (1u << 11) - 1,
+
+ LLVM_MARK_AS_BITMASK_ENUM(RESERVED_10)
+};
+
+/// The number of stages, which is also the number of bits in a stage mask.
+inline constexpr unsigned NumAsyncStages = BitWidth<AsyncStages>;
+
+/// The name of each stage, indexed by the position of its bit.
+inline constexpr StringLiteral AsyncStageNames[NumAsyncStages] = {
+ "TENSOR",
+ "GLOBAL_LOAD_ASYNC_TO_LDS",
+ "GLOBAL_LOAD_ASYNC_TO_LDS_MCAST",
+ "GLOBAL_STORE_ASYNC_FROM_LDS",
+ "RESERVED_4",
+ "BUFFER_GLOBAL_LOAD",
+ "RESERVED_6",
+ "RESERVED_7",
+ "RESERVED_8",
+ "RESERVED_9",
+ "RESERVED_10",
};
-/// Print the stages named by \p S as a '|'-separated list, or "all" if it
+/// \returns true if \p Mask is a legal stage mask operand of the asyncmark /
+/// wait_asyncmark intrinsics and pseudos.
+constexpr bool isValidAsyncStageMask(int64_t Mask) {
+ return Mask >= 0 && (Mask & ~int64_t(to_underlying(AsyncStages::ALL))) == 0;
+}
+
+/// \returns the stages named by the stage mask operand \p Mask of the
+/// asyncmark / wait_asyncmark intrinsics and pseudos. The zero mask is the one
+/// exception to a set bit naming a stage: it names every stage rather than
+/// none.
+constexpr AsyncStages asyncStagesFromMaskOperand(int64_t Mask) {
+ assert(isValidAsyncStageMask(Mask) && "Invalid async stage mask");
+ return Mask ? static_cast<AsyncStages>(Mask) : AsyncStages::ALL;
+}
+
+/// \returns the single stages named by \p Mask, from the least significant bit
+/// up.
+inline auto asyncStages(AsyncStages Mask) {
+ return make_filter_range(
+ map_range(seq(0u, NumAsyncStages),
+ [](unsigned I) { return static_cast<AsyncStages>(1u << I); }),
+ [Mask](AsyncStages Stage) { return any(Mask & Stage); });
+}
+
+/// \returns the position of the bit of the single stage \p Stage, for indexing
+/// per-stage data.
+inline unsigned getAsyncStageIndex(AsyncStages Stage) {
+ assert(has_single_bit(to_underlying(Stage)) && "Expected a single stage");
+ return countr_zero(to_underlying(Stage));
+}
+
+/// \returns the name of the single stage \p Stage.
+inline StringRef getAsyncStageName(AsyncStages Stage) {
+ return AsyncStageNames[getAsyncStageIndex(Stage)];
+}
+
+/// Print the stages named by \p Mask as a '|'-separated list, or "all" if it
/// names every stage.
-inline raw_ostream &operator<<(raw_ostream &OS, AsyncStages S) {
- if (S == AsyncStages::ALL)
+inline raw_ostream &operator<<(raw_ostream &OS, AsyncStages Mask) {
+ if (Mask == AsyncStages::ALL)
return OS << "all";
- if (S.none())
+ if (Mask == AsyncStages::NONE)
return OS << "none";
- interleave(S, OS, [&OS](AsyncStages Stage) { OS << Stage.getName(); }, "|");
+ interleave(
+ asyncStages(Mask), OS,
+ [&OS](AsyncStages Stage) { OS << getAsyncStageName(Stage); }, "|");
return OS;
}
-} // namespace AMDGPU
-} // namespace llvm
+} // namespace llvm::AMDGPU
#endif // LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 325ba441769e1a..78f4d0ea34ed33 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -380,7 +380,7 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
std::string Comment;
raw_string_ostream OS(Comment);
OS << " asyncmark(stages="
- << AMDGPU::AsyncStages::fromMaskOperand(MI->getOperand(0).getImm())
+ << AMDGPU::asyncStagesFromMaskOperand(MI->getOperand(0).getImm())
<< ')';
OutStreamer->emitRawComment(Comment);
}
@@ -392,7 +392,7 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
std::string Comment;
raw_string_ostream OS(Comment);
OS << " wait_asyncmark(" << MI->getOperand(0).getImm() << ", stages="
- << AMDGPU::AsyncStages::fromMaskOperand(MI->getOperand(1).getImm())
+ << AMDGPU::asyncStagesFromMaskOperand(MI->getOperand(1).getImm())
<< ')';
OutStreamer->emitRawComment(Comment);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
index dbac8bf66dfdef..5e36f0e08c02d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMIRFormatter.cpp
@@ -118,17 +118,17 @@ void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(uint64_t Imm,
void AMDGPUMIRFormatter::printAsyncStageMaskImm(int64_t Imm,
raw_ostream &OS) const {
- if (!AMDGPU::AsyncStages::isValidMask(Imm)) {
+ if (!AMDGPU::isValidAsyncStageMask(Imm)) {
OS << Imm;
return;
}
OS << AsyncStageImmPrefix;
- AMDGPU::AsyncStages Mask(Imm);
+ auto Mask = static_cast<AMDGPU::AsyncStages>(Imm);
// The zero mask names every stage. Spell that out rather than printing an
// empty list. The saturated mask means the same thing but is a different
// immediate, so it keeps its spelled-out list to stay round-trippable.
- if (Mask.none()) {
+ if (Mask == AMDGPU::AsyncStages::NONE) {
OS << AllStages;
return;
}
@@ -136,11 +136,13 @@ void AMDGPUMIRFormatter::printAsyncStageMaskImm(int64_t Imm,
// Reserved stages have no meaningful name, collect them into a single
// hex bitmask instead.
ListSeparator Delim(AsyncStageDelim);
- for (AMDGPU::AsyncStages S : Mask - AMDGPU::AsyncStages::RESERVED)
- OS << Delim << S.getName();
- if (AMDGPU::AsyncStages Reserved = Mask & AMDGPU::AsyncStages::RESERVED)
+ for (AMDGPU::AsyncStages S :
+ AMDGPU::asyncStages(Mask & ~AMDGPU::AsyncStages::RESERVED))
+ OS << Delim << AMDGPU::getAsyncStageName(S);
+ AMDGPU::AsyncStages Reserved = Mask & AMDGPU::AsyncStages::RESERVED;
+ if (any(Reserved))
OS << Delim << ReservedStagePrefix
- << format_hex_no_prefix(Reserved.value(), 1);
+ << format_hex_no_prefix(to_underlying(Reserved), 1);
}
void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI,
@@ -555,7 +557,7 @@ bool AMDGPUMIRFormatter::parseAsyncStageMaskImmMnemonic(
return false;
}
- AMDGPU::AsyncStages StageMask;
+ AMDGPU::AsyncStages StageMask = AMDGPU::AsyncStages::NONE;
while (!Src.empty()) {
StringRef Name = Src.substr(0, Src.find(AsyncStageDelim));
StringRef::iterator NamePos = Src.begin();
@@ -568,24 +570,23 @@ bool AMDGPUMIRFormatter::parseAsyncStageMaskImmMnemonic(
uint32_t Mask;
if (Bits.getAsInteger(16, Mask) || Bits.empty() || !Mask)
return ErrorCallback(NamePos, "invalid async stage mask");
- if (!AMDGPU::AsyncStages::isValidMask(Mask))
+ if (!AMDGPU::isValidAsyncStageMask(Mask))
return ErrorCallback(NamePos, "async stage mask out of range");
- if (Mask & ~AMDGPU::AsyncStages::RESERVED)
+ if (Mask & ~to_underlying(AMDGPU::AsyncStages::RESERVED))
return ErrorCallback(NamePos,
"async stage mask names a non-reserved stage");
- StageMask |= Mask;
+ StageMask |= static_cast<AMDGPU::AsyncStages>(Mask);
continue;
}
- auto Stages = AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL);
- auto It = find_if(
- Stages, [Name](AMDGPU::AsyncStages S) { return Name == S.getName(); });
- if (It == Stages.end())
+ const StringLiteral *It = find(AMDGPU::AsyncStageNames, Name);
+ if (It == std::end(AMDGPU::AsyncStageNames))
return ErrorCallback(NamePos, "invalid async stage name");
- StageMask |= *It;
+ StageMask |= static_cast<AMDGPU::AsyncStages>(
+ 1u << (It - std::begin(AMDGPU::AsyncStageNames)));
}
- Imm = StageMask.value();
+ Imm = to_underlying(StageMask);
return false;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index bb8f88dcc097ea..9b1b0b9fbbf0c8 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -821,7 +821,7 @@ class WaitcntBrackets {
SmallVector<const MachineInstr *> LDSDMAStores;
// State of all counters at each async mark encountered so far, per stage.
- std::array<SmallVector<CounterValueArray, 0>, AMDGPU::AsyncStages::NUM_STAGES>
+ std::array<SmallVector<CounterValueArray, 0>, AMDGPU::NumAsyncStages>
AsyncMarks;
// But in the rare pathological case, a nest of loops that pushes marks
@@ -832,7 +832,7 @@ class WaitcntBrackets {
// Track the upper bound score for async operations that are not part of a
// mark yet, per stage. Initialized to all zeros.
- std::array<CounterValueArray, AMDGPU::AsyncStages::NUM_STAGES> AsyncScore{};
+ std::array<CounterValueArray, AMDGPU::NumAsyncStages> AsyncScore{};
};
SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
@@ -1101,8 +1101,9 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
setVMemScore(LDSDMA_BEGIN + static_cast<VMEMID>(Slot), T, CurrScore);
}
- if (AMDGPU::AsyncStages Stage = Context->getAsyncStageToUpdate(Inst, T))
- AsyncScore[Stage.index()][T] = CurrScore;
+ AMDGPU::AsyncStages Stage = Context->getAsyncStageToUpdate(Inst, T);
+ if (any(Stage))
+ AsyncScore[AMDGPU::getAsyncStageIndex(Stage)][T] = CurrScore;
if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
setRegScore(AMDGPU::SCC, T, CurrScore);
@@ -1122,11 +1123,12 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
// A mark is inserted in every stage named. Each stage gets a copy of its
// corresponding score.
LLVM_DEBUG(dbgs() << "recordAsyncMark(stages=" << Stages << "):\n" << Inst);
- for (AMDGPU::AsyncStages S : Stages) {
- AsyncMarks[S.index()].push_back(AsyncScore[S.index()]);
+ for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Stages)) {
+ unsigned SI = AMDGPU::getAsyncStageIndex(S);
+ AsyncMarks[SI].push_back(AsyncScore[SI]);
LLVM_DEBUG({
dbgs() << " stage " << S << ":\n";
- for (const auto &Mark : AsyncMarks[S.index()]) {
+ for (const auto &Mark : AsyncMarks[SI]) {
dbgs() << " ";
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
@@ -1235,8 +1237,9 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
}
OS << '\n';
- for (AMDGPU::AsyncStages S : AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL)) {
- const CounterValueArray &Score = AsyncScore[S.index()];
+ for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(AMDGPU::AsyncStages::ALL)) {
+ unsigned SI = AMDGPU::getAsyncStageIndex(S);
+ const CounterValueArray &Score = AsyncScore[SI];
OS << "Async score (stage " << S << "): ";
if (llvm::all_of(Score, [](unsigned V) { return V == 0; }))
OS << "none";
@@ -1244,10 +1247,9 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
llvm::interleaveComma(Score, OS);
OS << '\n';
- OS << "Async marks (stage " << S << "): " << AsyncMarks[S.index()].size()
- << '\n';
+ OS << "Async marks (stage " << S << "): " << AsyncMarks[SI].size() << '\n';
- for (const auto &Mark : AsyncMarks[S.index()]) {
+ for (const auto &Mark : AsyncMarks[SI]) {
for (auto T : AMDGPU::inst_counter_types()) {
unsigned MarkedScore = Mark[T];
switch (T) {
@@ -1414,8 +1416,8 @@ WaitcntBrackets::determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages) {
// implied by the marks dropped from each stage accumulate into one Waitcnt,
// so a wait covering several stages is the union of their requirements.
AMDGPU::Waitcnt Wait;
- for (AMDGPU::AsyncStages S : Stages) {
- auto &StageMarks = AsyncMarks[S.index()];
+ for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Stages)) {
+ auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(S)];
LLVM_DEBUG({
dbgs() << "Need " << N << " async marks in stage " << S << ". Found "
<< StageMarks.size() << ":\n";
@@ -1751,7 +1753,7 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, AMDGPU::AsyncStages::fromMaskOperand(II.getOperand(1).getImm()));
+ N, AMDGPU::asyncStagesFromMaskOperand(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
@@ -2049,7 +2051,7 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
// shows up in the assembly as a comment with the original parameter N.
unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, AMDGPU::AsyncStages::fromMaskOperand(II.getOperand(1).getImm()));
+ N, AMDGPU::asyncStagesFromMaskOperand(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
std::optional<AMDGPU::InstCounterType> CT =
@@ -2816,7 +2818,7 @@ bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStages Stage,
ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks) {
bool StrictDom = false;
- auto &StageMarks = AsyncMarks[Stage.index()];
+ auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(Stage)];
LLVM_DEBUG(dbgs() << "Merging async marks of stage " << Stage << " ...");
// Early exit: nothing to merge when both sides are empty.
@@ -2973,11 +2975,12 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
- for (AMDGPU::AsyncStages S : AMDGPU::AsyncStages(AMDGPU::AsyncStages::ALL)) {
- StrictDom |= mergeAsyncMarks(S, MergeInfos, Other.AsyncMarks[S.index()]);
+ for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(AMDGPU::AsyncStages::ALL)) {
+ unsigned SI = AMDGPU::getAsyncStageIndex(S);
+ StrictDom |= mergeAsyncMarks(S, MergeInfos, Other.AsyncMarks[SI]);
for (auto T : inst_counter_types(Context->MaxCounter))
- StrictDom |= mergeScore(MergeInfos[T], AsyncScore[S.index()][T],
- Other.AsyncScore[S.index()][T]);
+ StrictDom |=
+ mergeScore(MergeInfos[T], AsyncScore[SI][T], Other.AsyncScore[SI][T]);
}
purgeEmptyTrackingData();
@@ -3156,7 +3159,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// Asyncmarks record the current wait state and so should not allow
// waitcnts that occur after them to be merged into waitcnts that occur
// before.
- ScoreBrackets.recordAsyncMark(Inst, AMDGPU::AsyncStages::fromMaskOperand(
+ ScoreBrackets.recordAsyncMark(Inst, AMDGPU::asyncStagesFromMaskOperand(
Inst.getOperand(0).getImm()));
continue;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 5ff8ebcca7fcca..8c9e0c7fdaa07f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5922,7 +5922,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
int64_t Mask = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
- if (!AMDGPU::AsyncStages::isValidMask(Mask)) {
+ if (!AMDGPU::isValidAsyncStageMask(Mask)) {
ErrInfo = "invalid asyncmark stage mask";
return false;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 7c7070d06b2e5d..8a6e58c0532419 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -39,7 +39,7 @@ using TargetRegisterClass = MCRegisterClass;
class ScheduleHazardRecognizer;
namespace AMDGPU {
-class AsyncStages;
+enum class AsyncStages : uint32_t;
} // namespace AMDGPU
constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
>From b9d16de2c3683d5d1aa6926dbc7813f934b1e277 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Thu, 29 Oct 2026 08:56:58 -0700
Subject: [PATCH 3/3] update tests after rebase
---
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 62 +++++++++----------
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 8 +--
2 files changed, 34 insertions(+), 36 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index d13af58c0f14e6..c2cb1fcdb7fe01 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -316,7 +316,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v7, 4
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, 4
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: s_add_co_i32 s7, s10, 4
@@ -325,12 +325,12 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0
; SDAG-NEXT: s_load_b32 s12, s[0:1], 0x0
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
-; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v8, v0, s[8:9] offset:4 nv
; SDAG-NEXT: ; asyncmark(stages=all)
-; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
-; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
-; SDAG-NEXT: global_load_async_to_lds_b32 v9, v7, s[8:9] offset:4 nv
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_b32 v2, v0, s[8:9] offset:4
+; SDAG-NEXT: global_load_b32 v3, v0, s[0:1] offset:4
+; SDAG-NEXT: global_load_async_to_lds_b32 v9, v5, s[8:9] offset:4 nv
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_mov_b32 s8, 2
@@ -346,18 +346,17 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
-; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v10, s9
-; SDAG-NEXT: global_load_b32 v3, v0, s[4:5]
-; SDAG-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_add_nc_u32 v11, v5, v6
-; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v9, s12
+; SDAG-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v10, s9
+; SDAG-NEXT: global_load_b32 v1, v0, s[4:5]
+; SDAG-NEXT: v_dual_add_nc_u32 v11, v6, v7 :: v_dual_mov_b32 v7, v3
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: global_load_b32 v4, v0, s[0:1]
; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: ds_load_b32 v10, v10
-; SDAG-NEXT: v_mov_b32_e32 v5, v1
+; SDAG-NEXT: v_mov_b32_e32 v6, v2
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
; SDAG-NEXT: s_cmp_lt_i32 s8, s11
@@ -382,15 +381,15 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: ds_load_b32 v2, v0
; SDAG-NEXT: v_mov_b32_e32 v5, s0
; SDAG-NEXT: s_add_co_i32 s0, s11, -1
-; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_dual_add_nc_u32 v1, v2, v1 :: v_dual_add_nc_u32 v2, v3, v4
-; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u32_e32 v2, v3, v2
+; SDAG-NEXT: global_store_b32 v5, v2, s[2:3] scale_offset
; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_dual_add_nc_u32 v0, v2, v0 :: v_dual_mov_b32 v2, s0
-; SDAG-NEXT: global_store_b32 v2, v0, s[2:3] scale_offset
+; SDAG-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_add_nc_u32 v0, v1, v0
+; SDAG-NEXT: global_store_b32 v1, v0, s[2:3] scale_offset
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_pipelined_loop_with_global:
@@ -404,18 +403,18 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
-; GISEL-NEXT: s_mov_b32 s12, 2
+; GISEL-NEXT: s_mov_b32 s13, 2
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_load_b32 s13, s[8:9], 0x0
+; GISEL-NEXT: s_load_b32 s12, s[8:9], 0x0
; GISEL-NEXT: s_load_b32 s16, s[0:1], 0x0
; GISEL-NEXT: v_mov_b32_e32 v1, s10
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v5, s6
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
-; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_b32 v2, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v3, v0, s[0:1] offset:4
; GISEL-NEXT: global_load_async_to_lds_b32 v5, v4, s[8:9] offset:4 nv
@@ -431,25 +430,24 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s15, v3
; GISEL-NEXT: s_mov_b32 s9, s8
-; GISEL-NEXT: s_mov_b32 s19, s15
+; GISEL-NEXT: s_mov_b32 s17, s15
; GISEL-NEXT: .LBB2_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: s_add_co_u32 s17, s14, 8
+; GISEL-NEXT: s_add_co_u32 s18, s14, 8
; GISEL-NEXT: global_load_b32 v2, v0, s[6:7]
-; GISEL-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v4, s14
-; GISEL-NEXT: s_mov_b32 s18, s9
+; GISEL-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v4, s14
+; GISEL-NEXT: s_mov_b32 s19, s9
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_add_co_i32 s9, s13, s16
-; GISEL-NEXT: s_add_co_i32 s12, s12, 1
-; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: s_add_co_i32 s9, s12, s16
+; GISEL-NEXT: s_add_co_i32 s13, s13, 1
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[6:7] offset:4 nv
; GISEL-NEXT: global_load_b32 v3, v0, s[0:1]
; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v4, v4
-; GISEL-NEXT: s_mov_b32 s17, s19
-; GISEL-NEXT: s_mov_b32 s13, s8
+; GISEL-NEXT: s_mov_b32 s18, s17
+; GISEL-NEXT: s_mov_b32 s12, s8
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s16, v4
; GISEL-NEXT: s_add_co_i32 s9, s9, s16
@@ -466,18 +464,18 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
; GISEL-NEXT: s_add_co_u32 s14, s14, 4
-; GISEL-NEXT: s_cmp_lt_i32 s12, s11
+; GISEL-NEXT: s_cmp_lt_i32 s13, s11
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s19, v3
+; GISEL-NEXT: v_readfirstlane_b32 s17, v3
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
-; GISEL-NEXT: s_add_co_i32 s4, s18, s17
+; GISEL-NEXT: s_add_co_i32 s4, s19, s18
; GISEL-NEXT: s_add_co_u32 s1, s10, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s1
@@ -488,7 +486,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_i32 s1, s4, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v2, s1
-; GISEL-NEXT: s_add_co_i32 s1, s9, s19
+; GISEL-NEXT: s_add_co_i32 s1, s9, s17
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 26a59bfa0dba2e..651bcc28fdbe33 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -501,8 +501,8 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark(stages=all)
-; SDAG-NEXT: global_load_dword v8, v[0:1], off
-; SDAG-NEXT: global_load_dword v9, v[3:4], off
+; SDAG-NEXT: global_load_dword v12, v[0:1], off
+; SDAG-NEXT: global_load_dword v15, v[3:4], off
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark(stages=all)
@@ -562,8 +562,8 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark(stages=all)
-; GISEL-NEXT: global_load_dword v8, v[0:1], off
-; GISEL-NEXT: global_load_dword v9, v[3:4], off
+; GISEL-NEXT: global_load_dword v12, v[0:1], off
+; GISEL-NEXT: global_load_dword v15, v[3:4], off
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark(stages=all)
More information about the Mlir-commits
mailing list