[clang] [llvm] [AMDGPU] Implement AsyncMark Stages (PR #220442)
Ryan Mitchell via cfe-commits
cfe-commits at lists.llvm.org
Tue Sep 8 19:00:52 PDT 2026
https://github.com/RyanRio updated https://github.com/llvm/llvm-project/pull/220442
>From 4e3b362719e7d41107ce4b39533cdf9fa047019c Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Tue, 1 Sep 2026 16:13:10 -0700
Subject: [PATCH 1/4] Asyncmark stage implementation
---
clang/include/clang/Basic/BuiltinsAMDGPU.td | 12 +-
.../clang/Basic/DiagnosticSemaKinds.td | 3 +
clang/lib/Sema/SemaAMDGPU.cpp | 33 ++
.../builtins-amdgcn-asyncmark-errs-gfx1250.cl | 33 ++
.../builtins-amdgcn-asyncmark-errs.cl | 4 +-
.../builtins-amdgcn-asyncmark.cl | 35 +-
llvm/docs/AMDGPUAsyncOperations.md | 134 +++++-
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 8 +-
llvm/include/llvm/Support/AMDGPUAsyncStages.h | 105 +++++
llvm/lib/IR/AutoUpgrade.cpp | 37 ++
.../AMDGPU/AMDGPUInstructionSelector.cpp | 13 +-
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 14 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 16 +
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 268 +++++++-----
llvm/lib/Target/AMDGPU/SOPInstructions.td | 8 +-
llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll | 33 ++
.../test/CodeGen/AMDGPU/async-buffer-loads.ll | 52 +--
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 138 +++---
.../CodeGen/AMDGPU/asyncmark-max-pregfx12.ll | 164 +++----
.../AMDGPU/asyncmark-merge-empty-other.mir | 16 +-
.../asyncmark-merge-rebase-pregfx12.mir | 16 +-
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 192 ++++-----
.../CodeGen/AMDGPU/asyncmark-stage-err.ll | 48 +++
.../CodeGen/AMDGPU/asyncmark-stage-max.mir | 218 ++++++++++
.../CodeGen/AMDGPU/asyncmark-stage-merge.mir | 284 +++++++++++++
.../AMDGPU/asyncmark-stage-pregfx12.ll | 115 +++++
llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll | 400 ++++++++++++++++++
.../test/CodeGen/AMDGPU/asyncmark-waitcnt.mir | 4 +-
.../CodeGen/AMDGPU/code-size-estimate.mir | 4 +-
.../AMDGPU/llvm.amdgcn.tensor.load.store.ll | 74 ++--
30 files changed, 2015 insertions(+), 466 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
create mode 100644 llvm/include/llvm/Support/AMDGPUAsyncStages.h
create mode 100644 llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 4e4e0ee275a16..a6682d69e8350 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -334,8 +334,16 @@ def __builtin_amdgcn_av_store_b128
//===----------------------------------------------------------------------===//
// Mirrors GCNSubtarget::hasAsyncMark()
-def __builtin_amdgcn_asyncmark : AMDGPUBuiltin<"void()", [], "vmem-to-lds-load-insts|asynccnt">;
-def __builtin_amdgcn_wait_asyncmark : AMDGPUBuiltin<"void(_Constant unsigned short)", [], "vmem-to-lds-load-insts|asynccnt">;
+def __builtin_amdgcn_asyncmark
+ : AMDGPUBuiltin<"void(_Constant unsigned int)", [],
+ "vmem-to-lds-load-insts|asynccnt"> {
+ let ArgNames = ["stage"];
+}
+def __builtin_amdgcn_wait_asyncmark
+ : AMDGPUBuiltin<"void(_Constant unsigned short, _Constant unsigned int)", [],
+ "vmem-to-lds-load-insts|asynccnt"> {
+ let ArgNames = ["num_marks", "stage"];
+}
//===----------------------------------------------------------------------===//
// Ballot builtins.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 24ecc88d2fbc0..07510286db89c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -14391,6 +14391,9 @@ def err_amdgcn_dmask_has_too_many_bits_set
: Error<"dmask argument cannot have more bits set than there are elements "
"in return type">;
+def err_amdgcn_asyncmark_stage_reserved
+ : Error<"asyncmark stage %0 is reserved">;
+
def warn_amdgpu_s_wait_event_mask_no_effect_target :
Warning<"event mask has no effect for target">,
InGroup<DiagGroup<"amdgpu-wait-event-mask">>;
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index 0bda851b0c1cf..b31085557e4a6 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -25,6 +25,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
#include <cstdint>
@@ -203,6 +204,38 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk16_f32_fp6:
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk16_f32_bf6:
return SemaRef.BuiltinConstantArgRange(TheCall, 2, 0, 15);
+ case AMDGPU::BI__builtin_amdgcn_asyncmark:
+ case AMDGPU::BI__builtin_amdgcn_wait_asyncmark: {
+ bool IsMark = BuiltinID == AMDGPU::BI__builtin_amdgcn_asyncmark;
+
+ // Check the sequence length limit is a constant.
+ llvm::APSInt NumMarks;
+ if (!IsMark && SemaRef.BuiltinConstantArg(TheCall, 0, NumMarks))
+ return true;
+
+ unsigned StageArgNum = IsMark ? 0 : 1;
+ llvm::APSInt Stage;
+ if (SemaRef.BuiltinConstantArg(TheCall, StageArgNum, Stage))
+ return true;
+
+ Expr *ArgExpr = TheCall->getArg(StageArgNum);
+ uint64_t Value = Stage.getZExtValue();
+
+ if (!llvm::AMDGPU::AsyncStage::isValidStage(Value)) {
+ return Diag(ArgExpr->getBeginLoc(), diag::err_argument_invalid_range)
+ << toString(Stage, 10) << 0 << (int)llvm::AMDGPU::AsyncStage::ALL
+ << ArgExpr->getSourceRange();
+ }
+
+ if (llvm::AMDGPU::AsyncStage::isReservedStage(Value)) {
+ return Diag(ArgExpr->getBeginLoc(),
+ diag::err_amdgcn_asyncmark_stage_reserved)
+ << llvm::AMDGPU::AsyncStage::getStageName(Value)
+ << ArgExpr->getSourceRange();
+ }
+
+ return false;
+ }
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
case AMDGPU::BI__builtin_amdgcn_av_store_b128:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
new file mode 100644
index 0000000000000..7889c1ef04700
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
@@ -0,0 +1,33 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.50-amd-amdhsa -verify -S -o - %s
+
+// The stage must be a compile-time constant; _Constant alone does not give a
+// range, so the value is checked here rather than left to the backend.
+
+void test_stage_not_constant(unsigned int s) {
+ __builtin_amdgcn_asyncmark(s); // expected-error{{argument to '__builtin_amdgcn_asyncmark' must be a constant integer}}
+ __builtin_amdgcn_wait_asyncmark(0, s); // expected-error{{argument to '__builtin_amdgcn_wait_asyncmark' must be a constant integer}}
+}
+
+// Stages above the catch-all, and the gap between the last named stage and the
+// catch-all, are out of range.
+
+void test_stage_out_of_range() {
+ __builtin_amdgcn_asyncmark(11); // expected-error{{argument value 11 is outside the valid range [0, 16]}}
+ __builtin_amdgcn_asyncmark(15); // expected-error{{argument value 15 is outside the valid range [0, 16]}}
+ __builtin_amdgcn_asyncmark(17); // expected-error{{argument value 17 is outside the valid range [0, 16]}}
+ __builtin_amdgcn_wait_asyncmark(0, 17); // expected-error{{argument value 17 is outside the valid range [0, 16]}}
+}
+
+// The reserved stages hold slots in the taxonomy for asynchronous operations
+// this target does not have. Using one is an error, not a silent no-op.
+
+void test_stage_reserved() {
+ __builtin_amdgcn_asyncmark(4); // expected-error{{asyncmark stage RESERVED_4 is reserved}}
+ __builtin_amdgcn_asyncmark(6); // expected-error{{asyncmark stage RESERVED_6 is reserved}}
+ __builtin_amdgcn_asyncmark(7); // expected-error{{asyncmark stage RESERVED_7 is reserved}}
+ __builtin_amdgcn_asyncmark(8); // expected-error{{asyncmark stage RESERVED_8 is reserved}}
+ __builtin_amdgcn_asyncmark(9); // expected-error{{asyncmark stage RESERVED_9 is reserved}}
+ __builtin_amdgcn_asyncmark(10); // expected-error{{asyncmark stage RESERVED_10 is reserved}}
+ __builtin_amdgcn_wait_asyncmark(0, 4); // expected-error{{asyncmark stage RESERVED_4 is reserved}}
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
index 63d68a7c30433..59ee951d98790 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
@@ -3,6 +3,6 @@
// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.00-amd-amdhsa -verify -S -o - %s
void test_feature() {
- __builtin_amdgcn_asyncmark(); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
- __builtin_amdgcn_wait_asyncmark(0); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_asyncmark(16); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_wait_asyncmark(0, 16); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
index 838bfdbdcb8f1..69750046e2e9e 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
@@ -5,13 +5,40 @@
// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -triple amdgpu12.50-unknown-unknown -emit-llvm -o - %s | FileCheck %s
// REQUIRES: amdgpu-registered-target
+// The stage argument selects which sequence of marks the builtin acts on, and
+// reaches the intrinsic unchanged. Stage 16 is the catch-all that observes
+// every counter.
+
// CHECK-LABEL: @test_invocation(
// CHECK-NEXT: entry:
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark()
-// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 16)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
// CHECK-NEXT: ret void
//
void test_invocation() {
- __builtin_amdgcn_asyncmark();
- __builtin_amdgcn_wait_asyncmark(0);
+ __builtin_amdgcn_asyncmark(16);
+ __builtin_amdgcn_wait_asyncmark(0, 16);
+}
+
+// Every supported stage is accepted.
+
+// CHECK-LABEL: @test_stages(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 1)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 3)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 5)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 5)
+// CHECK-NEXT: ret void
+//
+void test_stages() {
+ __builtin_amdgcn_asyncmark(0);
+ __builtin_amdgcn_asyncmark(1);
+ __builtin_amdgcn_asyncmark(2);
+ __builtin_amdgcn_asyncmark(3);
+ __builtin_amdgcn_asyncmark(5);
+ __builtin_amdgcn_wait_asyncmark(1, 0);
+ __builtin_amdgcn_wait_asyncmark(2, 5);
}
diff --git a/llvm/docs/AMDGPUAsyncOperations.md b/llvm/docs/AMDGPUAsyncOperations.md
index 4dc01148fe781..8154e5b443cd7 100644
--- a/llvm/docs/AMDGPUAsyncOperations.md
+++ b/llvm/docs/AMDGPUAsyncOperations.md
@@ -16,31 +16,66 @@ internally by the compiler. A thread that initiates one or more async operations
An *asyncmark* created by a thread can be used to track async operations
initiated by that thread.
+### Stages
+
+Every asyncmark belongs to a *stage*, which names a kind of async operation.
+Each async operation has an *own stage*, determined by the instruction that
+initiates it, and *belongs to* that stage and to the stage `ALL`.
+
+The stages are:
+
+| Value | Stage | Async operations |
+|---|---|---|
+| 0 | `TENSOR` | tensor loads and stores |
+| 1 | `GLOBAL_LOAD_ASYNC_TO_LDS` | global loads async to LDS |
+| 2 | `GLOBAL_LOAD_ASYNC_TO_LDS_MCAST` | multicast (cluster) global loads async to LDS |
+| 3 | `ASYNC_LDS_STORE` | async stores from LDS |
+| 5 | `UNFORMATTED_BUFFER_GLOBAL_LOAD` | unformatted buffer and global loads to LDS |
+| 16 | `ALL` | all of the above |
+
+The values not listed above are reserved for future async operations.
+Using one is an error.
+
+Which async operations a given subtarget actually has is described in
+{ref}`AMDGPU DMA Operations <amdgpu-dma-operations>`. A stage is valid on every
+subtarget that supports asyncmarks, whether or not that subtarget has any
+operation belonging to it; marking an empty stage is simply a no-op.
+
+Stage `ALL` is the catch-all: since every async operation belongs to it, an
+asyncmark in `ALL` tracks all async operations whatever their own stage.
+
### Current Sequence
-The abstract machine maintains a sequence of asyncmarks during the execution of
-a function body, which excludes any asyncmarks produced by calls to other
-functions encountered in the currently executing function. The state of this
-sequence at each program point in the function is called the *current sequence*.
+The abstract machine maintains a separate sequence of asyncmarks *for each
+stage* during the execution of a function body, which excludes any asyncmarks
+produced by calls to other functions encountered in the currently executing
+function. The state of the sequence for a stage `S` at each program point in
+the function is called the *current sequence of* `S`.
+
+The sequences of distinct stages are independent: appending to one does not
+affect the length or contents of any other.
-### `@llvm.amdgcn.asyncmark()`
+### `@llvm.amdgcn.asyncmark(i32 %S)`
-Produces an asyncmark and appends it to the current sequence.
+Produces an asyncmark in stage `S` and appends it to the current sequence of
+`S`. `S` must be a constant naming a stage that is not reserved.
-### `@llvm.amdgcn.wait.asyncmark(i16 %N)`
+### `@llvm.amdgcn.wait.asyncmark(i16 %N, i32 %S)`
-Ensures that the length of the current sequence is at most `N` by removing
-asyncmarks from the start of the sequence if it is more than `N`.
+Ensures that the length of the current sequence of `S` is at most `N` by
+removing asyncmarks from the start of that sequence if it is more than `N`. The
+sequences of other stages are unaffected. `S` must be a constant naming a stage
+that is not reserved.
### Completion of Asyncmarks
-An `asyncmark()` operation `X` that produces an asyncmark `M` is
-*completed-at* a `wait.asyncmark()` operation `Y` in the same function body
-if:
+An `asyncmark()` operation `X` that produces an asyncmark `M` in stage `S` is
+*completed-at* a `wait.asyncmark()` operation `Y` on stage `S` in the same
+function body if:
- `X` is *program-ordered* before `Y`, and
-- `M` is not in the current sequence at any operation `Z` that immediately
- follows `Y` in *program-order*.
+- `M` is not in the current sequence of `S` at any operation `Z` that
+ immediately follows `Y` in *program-order*.
## Completion of Async Operations
@@ -51,10 +86,14 @@ the thread can use an asyncmark to ensure that the async operation is
An async operation `A` *initiated-by* an instruction `I` is *completed-at* some
`wait.asyncmark()` operation `Y` if there exists an `asyncmark()` operation `X`
-such that:
+in a stage `S` such that:
+- `A` belongs to `S`,
- `I` is *program-ordered* before `X`, and
- `X` is *completed-at* `Y`.
+Since `A` belongs to `ALL` as well as to its own stage, an asyncmark and wait
+pair on `ALL` tracks `A` regardless of its own stage.
+
### happens-before
When an instruction `I` initiates an async operation `A`, `I` *happens-before*
@@ -65,6 +104,11 @@ If `A` is *completed-at* a `wait.asyncmark()` operation `Y`, then `A`
## Examples
+The stage argument is omitted in the examples below; they all use stage `ALL`,
+so every asyncmark tracks every async operation and there is only one sequence
+to reason about. The {ref}`interleaved stages <amdgpu-async-interleaved-stages>`
+example shows what stages add.
+
### Uneven blocks of async operations
```c++
@@ -151,6 +195,54 @@ void foo(global int *g, local int *l) {
}
```
+(amdgpu-async-interleaved-stages)=
+### Interleaved stages
+
+Two kinds of async operation are in flight at once. Each is marked in its own
+stage, so each can be waited for without waiting for the other.
+
+```c++
+void foo(global int *g, local int *l, tensor_desc t) {
+ // Start a long tensor load and mark it in the TENSOR stage.
+ tensor_load_to_lds(l, t);
+ asyncmark(TENSOR);
+
+ // Start a short LDS load and mark it in its own stage.
+ async_load_to_lds(l, g);
+ asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // Wait for the LDS load only. The tensor load is still in flight: its
+ // asyncmark is in a different sequence, so this wait neither counts nor
+ // removes it.
+ wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+
+ // perform synchronization / use the data loaded by async_load_to_lds
+
+ // Now wait for the tensor load.
+ wait.asyncmark(0, TENSOR);
+}
+```
+
+Had both marks been placed in `ALL`, the first wait would have drained the
+tensor load as well, and the overlap would have been lost.
+
+The same applies to two stages tracked by one hardware counter. Here both
+stages use the async counter, but the sequences are still separate:
+
+```c++
+void foo(global int *g, local int *l) {
+ async_load_to_lds(l, g);
+ asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS); // X
+
+ async_store_from_lds(g, l);
+ asyncmark(ASYNC_LDS_STORE); // Y
+
+ // Completes X. Y is in a different sequence and is not completed here, even
+ // though both stages are tracked by the same counter.
+ wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+}
+```
+
## Implementation notes
[This section is informational.]
@@ -233,10 +325,12 @@ After inlining, both `B` and `C` are *completed-at* `Y`.
### Optimization
The implementation may eliminate asyncmark/wait intrinsics in the following
-cases. These are just examples and not meant to be an exhaustive list.
+cases. These are just examples and not meant to be an exhaustive list. Each
+applies per stage: the sequences are independent, so an asyncmark in one stage
+is unaffected by the waits of another.
-1. An `asyncmark` operation which remains in the current sequence along every
- path that reaches the function exit.
+1. An `asyncmark` operation in a stage `S` which remains in the current
+ sequence of `S` along every path that reaches the function exit.
```c++
void foo() {
@@ -248,8 +342,8 @@ cases. These are just examples and not meant to be an exhaustive list.
Here, `X` can be eliminated.
-2. A `wait.asyncmark` which sees an empty sequence of asyncmarks along every
- path that reaches it.
+2. A `wait.asyncmark` on a stage `S` which sees an empty sequence of asyncmarks
+ for `S` along every path that reaches it.
```c++
void foo() {
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 037630ea86904..9ced72f61fe8d 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2923,12 +2923,14 @@ def int_amdgcn_pops_exiting_wave_id :
// Sets a marker in the stream of async requests. Modelled as InaccessibleMem.
def int_amdgcn_asyncmark : ClangBuiltin<"__builtin_amdgcn_asyncmark">,
- Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
+ Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
-// Waits until the Nth previous marker is completed, if it exists.
+// Waits until the Nth previous marker of the given stage is completed, if it
+// exists.
def int_amdgcn_wait_asyncmark :
ClangBuiltin<"__builtin_amdgcn_wait_asyncmark">,
- Intrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
+ Intrinsic<[], [llvm_i16_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, IntrNoMem, IntrHasSideEffects]>;
//===----------------------------------------------------------------------===//
// GFX10 Intrinsics
diff --git a/llvm/include/llvm/Support/AMDGPUAsyncStages.h b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
new file mode 100644
index 0000000000000..22a3f59b4317a
--- /dev/null
+++ b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
@@ -0,0 +1,105 @@
+//===- AMDGPUAsyncStages.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Shared AMDGPU asyncmark stage definitions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
+#define LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
+
+#include <cstdint>
+
+namespace llvm {
+namespace AMDGPU {
+namespace AsyncStage {
+
+// Stage selector for the asyncmark / wait_asyncmark intrinsics.
+// Do not renumber. Some values are RESERVED for later use.
+enum Stage : uint32_t {
+ // Tensor loads to LDS and tensor stores from LDS.
+ TENSOR = 0,
+ // Asynchronous global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS = 1,
+ // Asynchronous multicast (cluster) global loads to LDS.
+ GLOBAL_LOAD_ASYNC_TO_LDS_MCAST = 2,
+ // Asynchronous global stores from LDS.
+ ASYNC_LDS_STORE = 3,
+ RESERVED_4 = 4,
+ /// Non-format buffer loads to LDS and legacy global loads to LDS.
+ UNFORMATTED_BUFFER_GLOBAL_LOAD = 5,
+ RESERVED_6 = 6,
+ RESERVED_7 = 7,
+ RESERVED_8 = 8,
+ RESERVED_9 = 9,
+ RESERVED_10 = 10,
+ STAGE_LAST = RESERVED_10,
+
+ ALL = 16,
+};
+
+/// Number of distinct non-ALL stages. ALL is not part of this range.
+constexpr unsigned NumStages = STAGE_LAST + 1;
+
+/// Number of slots needed to index every stage, including ALL. The
+/// slots between STAGE_LAST and ALL are unused.
+constexpr unsigned NumStageSlots = ALL + 1;
+
+constexpr bool isValidStage(uint32_t S) { return S <= STAGE_LAST || S == ALL; }
+
+constexpr bool isReservedStage(uint32_t S) {
+ switch (S) {
+ case RESERVED_4:
+ case RESERVED_6:
+ case RESERVED_7:
+ case RESERVED_8:
+ case RESERVED_9:
+ case RESERVED_10:
+ return true;
+ default:
+ return false;
+ }
+}
+
+constexpr const char *getStageName(uint32_t S) {
+ switch (S) {
+ case TENSOR:
+ return "TENSOR";
+ case GLOBAL_LOAD_ASYNC_TO_LDS:
+ return "GLOBAL_LOAD_ASYNC_TO_LDS";
+ case GLOBAL_LOAD_ASYNC_TO_LDS_MCAST:
+ return "GLOBAL_LOAD_ASYNC_TO_LDS_MCAST";
+ case ASYNC_LDS_STORE:
+ return "ASYNC_LDS_STORE";
+ case RESERVED_4:
+ return "RESERVED_4";
+ case UNFORMATTED_BUFFER_GLOBAL_LOAD:
+ return "UNFORMATTED_BUFFER_GLOBAL_LOAD";
+ case RESERVED_6:
+ return "RESERVED_6";
+ case RESERVED_7:
+ return "RESERVED_7";
+ case RESERVED_8:
+ return "RESERVED_8";
+ case RESERVED_9:
+ return "RESERVED_9";
+ case RESERVED_10:
+ return "RESERVED_10";
+ case ALL:
+ return "ALL";
+ default:
+ return "invalid";
+ }
+}
+
+} // namespace AsyncStage
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index cd505273b5bfb..aa5ea83cad903 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -47,6 +47,7 @@
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
@@ -1582,6 +1583,19 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
switch (F->getIntrinsicID()) {
default:
break;
+ // Legacy asyncmark intrinsics without the stage operand.
+ case Intrinsic::amdgcn_asyncmark:
+ if (F->arg_size() == 0) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
+ case Intrinsic::amdgcn_wait_asyncmark:
+ if (F->arg_size() == 1) {
+ NewFn = nullptr;
+ return true;
+ }
+ break;
// Legacy wmma iu intrinsics without the optional clamp operand.
case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
if (F->arg_size() == 7) {
@@ -5120,6 +5134,29 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return NewCall;
};
+ // Legacy asyncmark intrinsics missed the stage operand. Append the ALL stage,
+ // which is the behavior they had: every async operation belongs to it.
+ if ((F->getIntrinsicID() == Intrinsic::amdgcn_asyncmark &&
+ CI->arg_size() == 0) ||
+ (F->getIntrinsicID() == Intrinsic::amdgcn_wait_asyncmark &&
+ CI->arg_size() == 1)) {
+ SmallVector<Value *, 2> Args(CI->args());
+ Args.push_back(Builder.getInt32(AMDGPU::AsyncStage::ALL));
+
+ Function *NewDecl =
+ Intrinsic::getOrInsertDeclaration(F->getParent(), F->getIntrinsicID());
+
+ SmallVector<OperandBundleDef, 1> Bundles;
+ CI->getOperandBundlesAsDefs(Bundles);
+
+ auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
+ NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ NewCall->setCallingConv(CI->getCallingConv());
+ NewCall->setAttributes(CI->getAttributes());
+ NewCall->copyMetadata(*CI);
+ return NewCall;
+ }
+
if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
"intrinsic should have 7 arguments");
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 893c7cce1a0f0..c7e6ba9fcb1c8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -26,6 +26,7 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include <optional>
#define DEBUG_TYPE "amdgpu-isel"
@@ -2438,10 +2439,20 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
case Intrinsic::amdgcn_tensor_store_from_lds:
return selectTensorLoadStore(I, IntrinsicID);
case Intrinsic::amdgcn_asyncmark:
- case Intrinsic::amdgcn_wait_asyncmark:
+ case Intrinsic::amdgcn_wait_asyncmark: {
if (!Subtarget->hasAsyncMark())
return false;
+ uint32_t Stage = I.getOperand(I.getNumOperands() - 1).getImm();
+ if (!AMDGPU::AsyncStage::isValidStage(Stage) ||
+ AMDGPU::AsyncStage::isReservedStage(Stage)) {
+ const Function &Fn = MF->getFunction();
+ Fn.getContext().diagnose(DiagnosticInfoUnsupported(
+ Fn, "intrinsic @llvm.amdgcn.*.asyncmark: invalid stage",
+ I.getDebugLoc(), DS_Error));
+ return false;
+ }
break;
+ }
case Intrinsic::amdgcn_ds_bvh_stack_rtn:
case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 41b93f49499ac..07d72a3cd8591 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -29,6 +29,7 @@
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCObjectStreamer.h"
#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Format.h"
@@ -373,15 +374,22 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
}
if (MI->getOpcode() == AMDGPU::ASYNCMARK) {
- if (isVerbose())
- OutStreamer->emitRawComment(" asyncmark");
+ if (isVerbose()) {
+ OutStreamer->emitRawComment(" asyncmark(" +
+ Twine(AMDGPU::AsyncStage::getStageName(
+ MI->getOperand(0).getImm())) +
+ ")");
+ }
return;
}
if (MI->getOpcode() == AMDGPU::WAIT_ASYNCMARK) {
if (isVerbose()) {
OutStreamer->emitRawComment(" wait_asyncmark(" +
- Twine(MI->getOperand(0).getImm()) + ")");
+ Twine(MI->getOperand(0).getImm()) + ", " +
+ Twine(AMDGPU::AsyncStage::getStageName(
+ MI->getOperand(1).getImm())) +
+ ")");
}
return;
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a8b4d1f9a568b..a27b8224f5063 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -45,6 +45,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
@@ -12558,6 +12559,21 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
initializeM0ToZeroForClusterLoad(Op, DAG, DL);
return SDValue();
}
+ case Intrinsic::amdgcn_asyncmark:
+ case Intrinsic::amdgcn_wait_asyncmark: {
+ Function &F = DAG.getMachineFunction().getFunction();
+ auto *StageC =
+ cast<ConstantSDNode>(Op->getOperand(Op.getNumOperands() - 1));
+ uint32_t StageRaw = StageC->getZExtValue();
+ if (!AMDGPU::AsyncStage::isValidStage(StageRaw) ||
+ AMDGPU::AsyncStage::isReservedStage(StageRaw)) {
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F, "intrinsic @llvm.amdgcn.*.asyncmark: invalid stage",
+ DL.getDebugLoc(), DS_Error));
+ return Chain;
+ }
+ return SDValue();
+ }
case Intrinsic::amdgcn_exp_compr: {
SDValue Src0 = Op.getOperand(4);
SDValue Src1 = Op.getOperand(5);
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index e4f40b0018eb1..40d894988355e 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -133,6 +134,14 @@ static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
}
}
+/// Interpret an asyncmark stage operand.
+static AMDGPU::AsyncStage::Stage getAsyncStage(int64_t Imm) {
+ if (Imm < 0 || !AMDGPU::AsyncStage::isValidStage(Imm) ||
+ AMDGPU::AsyncStage::isReservedStage(Imm))
+ return AMDGPU::AsyncStage::ALL;
+ return static_cast<AMDGPU::AsyncStage::Stage>(Imm);
+}
+
static bool updateVMCntOnly(const MachineInstr &Inst) {
return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
SIInstrInfo::isFLATGlobal(Inst) || SIInstrInfo::isFLATScratch(Inst);
@@ -413,15 +422,50 @@ class SIInsertWaitcnts {
return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
}
- bool shouldUpdateAsyncMark(const MachineInstr &MI,
- AMDGPU::InstCounterType T) const {
- if (SIInstrInfo::usesTENSOR_CNT(MI))
- return T == AMDGPU::TENSOR_CNT;
+ std::optional<AMDGPU::AsyncStage::Stage>
+ shouldUpdateAsyncMark(const MachineInstr &MI,
+ AMDGPU::InstCounterType T) const {
+ if (SIInstrInfo::usesTENSOR_CNT(MI) && T == AMDGPU::TENSOR_CNT)
+ return AMDGPU::AsyncStage::TENSOR;
if (!isAsyncLdsDmaWrite(MI))
- return false;
- if (SIInstrInfo::usesASYNC_CNT(MI))
- return T == AMDGPU::ASYNC_CNT;
- return T == AMDGPU::LOAD_CNT;
+ return std::nullopt;
+ if (SIInstrInfo::usesASYNC_CNT(MI) && T == AMDGPU::ASYNC_CNT) {
+ switch (MI.getOpcode()) {
+ // Keep in sync with FLATInstructions.td.
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ return AMDGPU::AsyncStage::GLOBAL_LOAD_ASYNC_TO_LDS;
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ return AMDGPU::AsyncStage::GLOBAL_LOAD_ASYNC_TO_LDS_MCAST;
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128:
+ case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR:
+ return AMDGPU::AsyncStage::ASYNC_LDS_STORE;
+ default:
+ llvm_unreachable("Async opcode has no associated async stage");
+ }
+ }
+ if (!SIInstrInfo::usesASYNC_CNT(MI) && T == AMDGPU::LOAD_CNT)
+ return AMDGPU::AsyncStage::UNFORMATTED_BUFFER_GLOBAL_LOAD;
+ return std::nullopt;
}
bool isVmemAccess(const MachineInstr &MI) const;
@@ -561,14 +605,15 @@ class WaitcntBrackets {
MCPhysReg Reg) const;
void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
AMDGPU::Waitcnt &Wait) const;
- AMDGPU::Waitcnt determineAsyncWait(unsigned N);
+ AMDGPU::Waitcnt determineAsyncWait(unsigned N,
+ AMDGPU::AsyncStage::Stage Stage);
void tryClearSCCWriteEvent(MachineInstr *Inst);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
void updateByEvent(HWEvents E, MachineInstr &MI);
- void recordAsyncMark(MachineInstr &MI);
+ void recordAsyncMark(MachineInstr &MI, AMDGPU::AsyncStage::Stage Stage);
HWEvents getPendingEvents() const { return PendingEvents; }
bool hasPendingEvent() const { return PendingEvents.any(); }
@@ -671,7 +716,8 @@ class WaitcntBrackets {
static bool mergeScore(const MergeInfo &M, unsigned &Score,
unsigned OtherScore);
- bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
+ bool mergeAsyncMarks(AMDGPU::AsyncStage::Stage Stage,
+ ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks);
iterator_range<MCRegUnitIterator> regunits(MCPhysReg Reg) const {
@@ -792,8 +838,10 @@ class WaitcntBrackets {
// alias info. One store is kept per unique AAInfo.
SmallVector<const MachineInstr *> LDSDMAStores;
- // State of all counters at each async mark encountered so far.
- SmallVector<CounterValueArray> AsyncMarks;
+ // State of all counters at each async mark encountered so far, per stage.
+ std::array<SmallVector<CounterValueArray, 0>,
+ AMDGPU::AsyncStage::NumStageSlots>
+ AsyncMarks;
// But in the rare pathological case, a nest of loops that pushes marks
// without waiting on any mark can cause AsyncMarks to grow very large. We cap
@@ -802,8 +850,8 @@ class WaitcntBrackets {
static constexpr unsigned MaxAsyncMarks = 16;
// Track the upper bound score for async operations that are not part of a
- // mark yet. Initialized to all zeros.
- CounterValueArray AsyncScore{};
+ // mark yet, per stage. Initialized to all zeros.
+ std::array<CounterValueArray, AMDGPU::AsyncStage::NumStageSlots> AsyncScore{};
};
SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
@@ -1072,8 +1120,9 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
setVMemScore(LDSDMA_BEGIN + Slot, T, CurrScore);
}
- if (Context->shouldUpdateAsyncMark(Inst, T)) {
- AsyncScore[T] = CurrScore;
+ if (auto Stage = Context->shouldUpdateAsyncMark(Inst, T)) {
+ AsyncScore[AMDGPU::AsyncStage::ALL][T] = CurrScore;
+ AsyncScore[*Stage][T] = CurrScore;
}
if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
@@ -1083,16 +1132,19 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
}
}
-void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
+void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
+ AMDGPU::AsyncStage::Stage Stage) {
// In the absence of loops, AsyncMarks can grow linearly with the program
// until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
// limit every time we push a new mark, but that seems like unnecessary work
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
- AsyncMarks.push_back(AsyncScore);
+ AsyncMarks[Stage].push_back(AsyncScore[Stage]);
LLVM_DEBUG({
- dbgs() << "recordAsyncMark:\n" << Inst;
- for (const auto &Mark : AsyncMarks) {
+ dbgs() << "recordAsyncMark(" << AMDGPU::AsyncStage::getStageName(Stage)
+ << "):\n"
+ << Inst;
+ for (const auto &Mark : AsyncMarks[Stage]) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
@@ -1199,55 +1251,60 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
}
OS << '\n';
- OS << "Async score: ";
- if (AsyncScore.empty())
- OS << "none";
- else
- llvm::interleaveComma(AsyncScore, OS);
- OS << '\n';
+ for (unsigned S = 0; S < AMDGPU::AsyncStage::NumStageSlots; ++S) {
+ if (!AMDGPU::AsyncStage::isValidStage(S))
+ continue;
+ OS << "Async score (stage " << AMDGPU::AsyncStage::getStageName(S) << "): ";
+ if (llvm::all_of(AsyncScore[S], [](unsigned V) { return V == 0; }))
+ OS << "none";
+ else
+ llvm::interleaveComma(AsyncScore[S], OS);
+ OS << '\n';
- OS << "Async marks: " << AsyncMarks.size() << '\n';
+ OS << "Async marks (stage " << AMDGPU::AsyncStage::getStageName(S)
+ << "): " << AsyncMarks[S].size() << '\n';
- for (const auto &Mark : AsyncMarks) {
- for (auto T : AMDGPU::inst_counter_types()) {
- unsigned MarkedScore = Mark[T];
- switch (T) {
- case AMDGPU::LOAD_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::DS_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::EXP_CNT:
- OS << " EXP_CNT: " << MarkedScore;
- break;
- case AMDGPU::STORE_CNT:
- OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
- << "_CNT: " << MarkedScore;
- break;
- case AMDGPU::SAMPLE_CNT:
- OS << " SAMPLE_CNT: " << MarkedScore;
- break;
- case AMDGPU::BVH_CNT:
- OS << " BVH_CNT: " << MarkedScore;
- break;
- case AMDGPU::KM_CNT:
- OS << " KM_CNT: " << MarkedScore;
- break;
- case AMDGPU::X_CNT:
- OS << " X_CNT: " << MarkedScore;
- break;
- case AMDGPU::ASYNC_CNT:
- OS << " ASYNC_CNT: " << MarkedScore;
- break;
- default:
- OS << " UNKNOWN: " << MarkedScore;
- break;
+ for (const auto &Mark : AsyncMarks[S]) {
+ for (auto T : AMDGPU::inst_counter_types()) {
+ unsigned MarkedScore = Mark[T];
+ switch (T) {
+ case AMDGPU::LOAD_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::DS_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::EXP_CNT:
+ OS << " EXP_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::STORE_CNT:
+ OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
+ << "_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::SAMPLE_CNT:
+ OS << " SAMPLE_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::BVH_CNT:
+ OS << " BVH_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::KM_CNT:
+ OS << " KM_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::X_CNT:
+ OS << " X_CNT: " << MarkedScore;
+ break;
+ case AMDGPU::ASYNC_CNT:
+ OS << " ASYNC_CNT: " << MarkedScore;
+ break;
+ default:
+ OS << " UNKNOWN: " << MarkedScore;
+ break;
+ }
}
+ OS << '\n';
}
- OS << '\n';
}
OS << '\n';
}
@@ -1368,17 +1425,21 @@ void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
}
}
-AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
+AMDGPU::Waitcnt
+WaitcntBrackets::determineAsyncWait(unsigned N,
+ AMDGPU::AsyncStage::Stage Stage) {
+ auto &StageMarks = AsyncMarks[Stage];
LLVM_DEBUG({
- dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
- << ":\n";
- for (const auto &Mark : AsyncMarks) {
+ dbgs() << "Need " << N << " async marks in stage "
+ << AMDGPU::AsyncStage::getStageName(Stage) << ". Found "
+ << StageMarks.size() << ":\n";
+ for (const auto &Mark : StageMarks) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
});
- if (AsyncMarks.size() == MaxAsyncMarks) {
+ if (StageMarks.size() == MaxAsyncMarks) {
// Enforcing MaxAsyncMarks here is unnecessary work because the size of
// MaxAsyncMarks is linear when traversing straightline code. But we do
// need to check if truncation may have occured at a merge, and adjust N
@@ -1388,15 +1449,16 @@ AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
}
AMDGPU::Waitcnt Wait;
- if (AsyncMarks.size() <= N) {
+ if (StageMarks.size() <= N) {
LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
return Wait;
}
- size_t MarkIndex = AsyncMarks.size() - N - 1;
- const auto &RequiredMark = AsyncMarks[MarkIndex];
- for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
+ size_t MarkIndex = StageMarks.size() - N - 1;
+ const auto &RequiredMark = StageMarks[MarkIndex];
+ for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
determineWaitForScore(T, RequiredMark[T], Wait);
+ }
// Immediately remove the waited mark and all older ones
// This happens BEFORE the wait is actually inserted, which is fine
@@ -1405,7 +1467,7 @@ AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
dbgs() << "Removing " << (MarkIndex + 1)
<< " async marks after determining wait\n";
});
- AsyncMarks.erase(AsyncMarks.begin(), AsyncMarks.begin() + MarkIndex + 1);
+ StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
return Wait;
@@ -1692,7 +1754,8 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
} else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
unsigned N = II.getOperand(0).getImm();
LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
- AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
+ AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
+ N, getAsyncStage(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
@@ -1983,7 +2046,8 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
// Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
// shows up in the assembly as a comment with the original parameter N.
unsigned N = II.getOperand(0).getImm();
- AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
+ AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
+ N, getAsyncStage(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
std::optional<AMDGPU::InstCounterType> CT =
@@ -2749,37 +2813,40 @@ bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
return OtherShifted > MyShifted;
}
-bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
+bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStage::Stage Stage,
+ ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks) {
bool StrictDom = false;
+ auto &StageMarks = AsyncMarks[Stage];
- LLVM_DEBUG(dbgs() << "Merging async marks ...");
+ LLVM_DEBUG(dbgs() << "Merging async marks of stage "
+ << AMDGPU::AsyncStage::getStageName(Stage) << " ...");
// Early exit: nothing to merge when both sides are empty.
- if (AsyncMarks.empty() && OtherMarks.empty()) {
+ if (StageMarks.empty() && OtherMarks.empty()) {
LLVM_DEBUG(dbgs() << " nothing to merge\n");
return false;
}
LLVM_DEBUG(dbgs() << '\n');
// Determine maximum length needed after merging
- auto MaxSize = (unsigned)std::max(AsyncMarks.size(), OtherMarks.size());
+ auto MaxSize = (unsigned)std::max(StageMarks.size(), OtherMarks.size());
MaxSize = std::min(MaxSize, MaxAsyncMarks);
// Keep only the most recent marks within our limit.
- if (AsyncMarks.size() > MaxSize)
- AsyncMarks.erase(AsyncMarks.begin(),
- AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
+ if (StageMarks.size() > MaxSize)
+ StageMarks.erase(StageMarks.begin(),
+ StageMarks.begin() + (StageMarks.size() - MaxSize));
// Pad with zero-filled marks if our list is shorter. Zero represents "no
// pending async operations at this checkpoint" and acts as the identity
// element for max() during merging. We pad at the beginning since the marks
// need to be aligned in most-recent order.
constexpr CounterValueArray ZeroMark{};
- AsyncMarks.insert(AsyncMarks.begin(), MaxSize - AsyncMarks.size(), ZeroMark);
+ StageMarks.insert(StageMarks.begin(), MaxSize - StageMarks.size(), ZeroMark);
LLVM_DEBUG({
dbgs() << "Before merge:\n";
- for (const auto &Mark : AsyncMarks) {
+ for (const auto &Mark : StageMarks) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
@@ -2798,26 +2865,26 @@ bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
// re-expressed in the new frame instead of being left with a stale
// (pre-merge) score.
const unsigned OtherSize = OtherMarks.size();
- const unsigned OurSize = AsyncMarks.size();
- // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
- // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
+ const unsigned OurSize = StageMarks.size();
+ // After the both-empty early-return above, max(StageMarks, OtherMarks) >= 1,
+ // and the erase/pad steps normalize StageMarks to exactly MaxSize. Hence
// OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
// seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
// the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
assert(OurSize >= 1 &&
- "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
+ "StageMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
for (auto Idx : seq_inclusive<unsigned>(1, OurSize)) {
const CounterValueArray &OtherMark =
Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
for (auto T : inst_counter_types(Context->MaxCounter))
StrictDom |=
- mergeScore(MergeInfos[T], AsyncMarks[OurSize - Idx][T], OtherMark[T]);
+ mergeScore(MergeInfos[T], StageMarks[OurSize - Idx][T], OtherMark[T]);
}
LLVM_DEBUG({
dbgs() << "After merge:\n";
- for (const auto &Mark : AsyncMarks) {
+ for (const auto &Mark : StageMarks) {
llvm::interleaveComma(Mark, dbgs());
dbgs() << '\n';
}
@@ -2908,9 +2975,15 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
- StrictDom |= mergeAsyncMarks(MergeInfos, Other.AsyncMarks);
- for (auto T : inst_counter_types(Context->MaxCounter))
- StrictDom |= mergeScore(MergeInfos[T], AsyncScore[T], Other.AsyncScore[T]);
+ for (unsigned S = 0; S != AMDGPU::AsyncStage::NumStageSlots; ++S) {
+ if (!AMDGPU::AsyncStage::isValidStage(S))
+ continue;
+ auto Stage = static_cast<AMDGPU::AsyncStage::Stage>(S);
+ StrictDom |= mergeAsyncMarks(Stage, MergeInfos, Other.AsyncMarks[S]);
+ for (auto T : inst_counter_types(Context->MaxCounter))
+ StrictDom |=
+ mergeScore(MergeInfos[T], AsyncScore[S][T], Other.AsyncScore[S][T]);
+ }
purgeEmptyTrackingData();
return StrictDom;
@@ -3088,7 +3161,8 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// Asyncmarks record the current wait state and so should not allow
// waitcnts that occur after them to be merged into waitcnts that occur
// before.
- ScoreBrackets.recordAsyncMark(Inst);
+ ScoreBrackets.recordAsyncMark(Inst,
+ getAsyncStage(Inst.getOperand(0).getImm()));
continue;
}
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 2c5a2b1f7d0d7..25bbe27f55281 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1725,13 +1725,13 @@ def S_WAITCNT_lds_direct : SPseudoInstSI<(outs), (ins)> {
}
let SubtargetPredicate = HasAsyncMark in {
-def ASYNCMARK : SPseudoInstSI<(outs), (ins),
- [(int_amdgcn_asyncmark)]> {
+def ASYNCMARK : SPseudoInstSI<(outs), (ins i32imm:$stage),
+ [(int_amdgcn_asyncmark timm:$stage)]> {
let maybeAtomic = 0;
let isMeta = 1;
}
-def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16), "$simm16",
- [(int_amdgcn_wait_asyncmark timm:$simm16)]> {
+def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16, i32imm:$stage),
+ "$simm16", [(int_amdgcn_wait_asyncmark timm:$simm16, timm:$stage)]> {
let maybeAtomic = 0;
let Size = 0;
let isMeta = 1;
diff --git a/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
new file mode 100644
index 0000000000000..4997ad6d4c06d
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
@@ -0,0 +1,33 @@
+; RUN: split-file %s %t
+; RUN: llvm-as < %t/legacy.ll | llvm-dis | FileCheck %s --check-prefix=LEGACY
+; RUN: llvm-as < %t/staged.ll | llvm-dis | FileCheck %s --check-prefix=STAGED
+
+; The asyncmark intrinsics originally had no stage operand. Upgrade them to the
+; ALL stage (16), which is the behavior they had.
+
+;--- legacy.ll
+define void @legacy() {
+; LEGACY-LABEL: define void @legacy(
+; LEGACY-NEXT: call void @llvm.amdgcn.asyncmark(i32 16)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
+; LEGACY-NEXT: ret void
+;
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ ret void
+}
+
+;--- staged.ll
+; Calls that already carry a stage operand are left alone.
+define void @staged() {
+; STAGED-LABEL: define void @staged(
+; STAGED-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; STAGED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+; STAGED-NEXT: ret void
+;
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
index 2d096c5af4cc5..2d0248dd9ae49 100644
--- a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
@@ -9,23 +9,23 @@ define float @raw.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %lds
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -37,23 +37,23 @@ define float @raw.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace(3)
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -65,23 +65,23 @@ define float @struct.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -93,23 +93,23 @@ define float @struct.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1)
+; CHECK-NEXT: ; wait_asyncmark(1, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -124,13 +124,13 @@ define float @struct.ptr.buffer.load.vgpr.lds(ptr addrspace(8) inreg %rsrc, ptr
; CHECK-NEXT: s_mov_b32 m0, s4
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v1, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark
+; CHECK-NEXT: ; asyncmark(ALL)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index d6089f85b8a4b..9c4cf0e5825b4 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -21,18 +21,18 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
; SDAG-NEXT: ; wave barrier
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
; SDAG-NEXT: s_wait_loadcnt 0x2
@@ -60,7 +60,7 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
; GISEL-NEXT: ; wave barrier
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
@@ -69,11 +69,11 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
; GISEL-NEXT: s_wait_loadcnt 0x2
@@ -96,7 +96,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
call void @llvm.amdgcn.wave.barrier()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Second batch: global load, async global-to-LDS, global load
%foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
@@ -108,15 +108,15 @@ entry:
call void @llvm.amdgcn.wave.barrier()
%bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
%bar_v12 = load i32, ptr addrspace(1) %bar_gep12
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for first async mark and read from LDS
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val21 = load i32, ptr addrspace(3) %lds_gep21
; Wait for the next async mark.
; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val22 = load i32, ptr addrspace(3) %lds_gep22
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -151,11 +151,11 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b32 s7, s2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: .LBB1_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
@@ -163,8 +163,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: v_mov_b32_e32 v2, s8
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v2, s7
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(2, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s7, s7, 4
; SDAG-NEXT: s_cmp_lt_i32 s6, s3
@@ -175,14 +175,14 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_add_co_i32 s0, s0, -8
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; SDAG-NEXT: ds_load_b32 v0, v0
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
@@ -208,11 +208,11 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: s_mov_b32 s8, s2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: .LBB1_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
@@ -220,8 +220,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: v_mov_b32_e32 v1, s9
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v1, s8
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(2, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v1, v1
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -234,7 +234,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_lshl_b32 s0, s3, 2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_add_co_u32 s0, s2, s0
; GISEL-NEXT: v_mov_b32_e32 v1, 0
@@ -243,7 +243,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GISEL-NEXT: ds_load_b32 v0, v0
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s2, v0
@@ -256,13 +256,13 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Load second iteration
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_body
@@ -274,10 +274,10 @@ loop_body:
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -290,14 +290,14 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
%sum_e2 = add i32 %sum_i, %lds_val_n_2
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -330,7 +330,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
@@ -338,7 +338,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_mov_b32 s8, 2
; SDAG-NEXT: s_mov_b32 s9, s10
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s12
; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
@@ -357,8 +357,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(2, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
@@ -376,7 +376,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_add_co_i32 s0, s11, -2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -387,7 +387,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v1, v2, v1
; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
; SDAG-NEXT: s_wait_loadcnt 0x0
@@ -421,7 +421,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_mov_b32 s14, s10
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; GISEL-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
@@ -431,7 +431,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
; GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
; GISEL-NEXT: s_add_co_ci_u32 s7, s9, 0
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s15, v1
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -452,8 +452,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_mov_b32 s9, s19
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[6:7] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v3, s14
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(2, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: s_mov_b32 s12, s15
; GISEL-NEXT: ds_load_b32 v3, v3
@@ -480,7 +480,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
; GISEL-NEXT: s_add_co_i32 s4, s18, s9
@@ -496,7 +496,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: v_mov_b32_e32 v2, s1
; GISEL-NEXT: s_add_co_i32 s1, s8, s19
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0 offset:4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -512,7 +512,7 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Load second iteration
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
@@ -522,7 +522,7 @@ prolog:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_body
@@ -540,10 +540,10 @@ loop_body:
%cur_g = load i32, ptr addrspace(1) %bar_gep_cur
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -559,7 +559,7 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
@@ -568,7 +568,7 @@ epilog:
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
store i32 %sum_e2, ptr addrspace(1) %out_gep_e1
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -589,9 +589,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -604,9 +604,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
@@ -615,9 +615,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -636,11 +636,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_add_nc_u32_e32 v3, 4, v2
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -655,11 +655,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v2
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -668,11 +668,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
index 79017c052d0cb..7ed261f9f4d34 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
@@ -10,26 +10,26 @@
; Loop body: 18 markers
; CHECK-LABEL: test_loop_exceeds_max_first_iteration:
-; CHECK: ; wait_asyncmark(3)
+; CHECK: ; wait_asyncmark(3, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_exceeds_max_first_iteration(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_header
@@ -42,49 +42,49 @@ loop_header:
loop_body:
; Loop body with 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -95,26 +95,26 @@ exit:
; Loop body: 5 markers
; CHECK-LABEL: test_loop_needs_more_iterations:
-; CHECK: ; wait_asyncmark(3)
+; CHECK: ; wait_asyncmark(3, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_needs_more_iterations(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_header
@@ -127,20 +127,20 @@ loop_header:
loop_body:
; Loop body with 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -149,7 +149,7 @@ exit:
; Merge exceeds MaxAsyncMarkers
; CHECK-LABEL: max_when_merged:
-; CHECK: ; wait_asyncmark(17)
+; CHECK: ; wait_asyncmark(17, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(15)
define void @max_when_merged(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
@@ -160,68 +160,68 @@ entry:
then:
; 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %endif
else:
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %endif
endif:
- call void @llvm.amdgcn.wait.asyncmark(i16 17)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 16)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -230,52 +230,52 @@ endif:
; Straightline exceeds MaxAsyncMarkers
; CHECK-LABEL: no_max_in_straightline:
-; CHECK: ; wait_asyncmark(17)
+; CHECK: ; wait_asyncmark(17, ALL)
; CHECK-NEXT: s_waitcnt vmcnt(17)
define void @no_max_in_straightline(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 17)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 16)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
index 3282ba05896fc..128c3432e8f70 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-empty-other.mir
@@ -43,7 +43,7 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK 16
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -52,7 +52,7 @@ body: |
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -66,7 +66,7 @@ body: |
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 16
S_BRANCH %bb.3
; else branch — sync path, no async operations; OtherMarks is empty at join
@@ -76,7 +76,7 @@ body: |
; join — mergeAsyncMarks sees non-empty AsyncMarks (then) and empty OtherMarks (else).
; Before fix: assertion. After fix: returns early, no spurious wait inserted.
bb.3:
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 16
S_ENDPGM 0
...
---
@@ -107,11 +107,11 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK 16
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -129,11 +129,11 @@ body: |
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 16
S_BRANCH %bb.3
; join block
bb.3:
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 16
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
index dd41980f42b14..f39c38fee121c 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
@@ -39,13 +39,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $m0 = S_MOV_B32 0
; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK 16
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_ENDPGM 0
@@ -75,13 +75,13 @@ body: |
$m0 = S_MOV_B32 0
GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 16
S_BRANCH %bb.3
bb.3:
liveins: $vgpr2
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 16
renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
S_ENDPGM 0
...
@@ -107,7 +107,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $m0 = S_MOV_B32 0
; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK 16
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -124,7 +124,7 @@ body: |
; CHECK-NEXT: bb.3:
; CHECK-NEXT: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_ENDPGM 0
@@ -142,7 +142,7 @@ body: |
$m0 = S_MOV_B32 0
GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
+ ASYNCMARK 16
S_BRANCH %bb.3
; branch target: load predecessor, merged first (seeds incoming state)
@@ -160,7 +160,7 @@ body: |
bb.3:
liveins: $vgpr2
- WAIT_ASYNCMARK 0
+ WAIT_ASYNCMARK 0, 16
renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 3d8f0b94c24a7..110935538509f 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -13,7 +13,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: ds_read_b32 v1, v2 offset:4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v0, v1
@@ -25,7 +25,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: ds_read_b32 v1, v2 offset:4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v0, v1
@@ -34,7 +34,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL-NEXT: s_setpc_b64 s[30:31]
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%val1 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%val2 = load i32, ptr addrspace(3) %lds_gep1
%sum = add i32 %val1, %val2
store i32 %sum, ptr addrspace(3) %out
@@ -58,18 +58,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[3:4], off
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v4, v8, v7
@@ -91,18 +91,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[3:4], off
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v4, v8, v7
@@ -119,7 +119,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Second batch: global load, async global-to-LDS, global load
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -127,16 +127,16 @@ entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for first async mark and read from LDS
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma
; This results in vmcnt(1), corresponding to %bar_v12. Could have been combined with the lgkmcnt(1) for %lds_val21.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -160,7 +160,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: v_mov_b32_e32 v8, 0x54
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -168,11 +168,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -193,7 +193,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: v_mov_b32_e32 v8, 0x54
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -201,11 +201,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -222,7 +222,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -230,16 +230,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -264,7 +264,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -272,10 +272,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -297,7 +297,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -305,10 +305,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -326,7 +326,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
fence release
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -334,16 +334,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -368,20 +368,20 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: .LBB4_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s7, v2
; SDAG-NEXT: s_mov_b32 m0, s7
; SDAG-NEXT: s_add_i32 s6, s6, 1
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(2, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: ds_read_b32 v6, v2
; SDAG-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -392,10 +392,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_cbranch_execnz .LBB4_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v5, v0
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -409,21 +409,21 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 s7, 0
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v5, s7
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: .LBB4_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: v_readfirstlane_b32 s7, v2
; GISEL-NEXT: s_mov_b32 m0, s7
; GISEL-NEXT: s_add_i32 s6, s6, 1
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(2, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: ds_read_b32 v6, v2
; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -434,10 +434,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v5, v0
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -446,11 +446,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Load second iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_body
@@ -460,10 +460,10 @@ loop_body:
; Load next iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -475,11 +475,11 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e2 = add i32 %sum_i, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e1 = add i32 %sum_e2, %lds_val_n_1
store i32 %sum_e2, ptr addrspace(1) %bar
@@ -500,12 +500,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: global_load_dword v14, v[3:4], off
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: global_load_dword v8, v[0:1], off
; SDAG-NEXT: global_load_dword v9, v[3:4], off
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_mov_b32_e32 v13, v8
; SDAG-NEXT: s_waitcnt vmcnt(1)
@@ -527,17 +527,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: v_mov_b32_e32 v10, v8
; SDAG-NEXT: v_mov_b32_e32 v14, v9
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(2, ALL)
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_cbranch_execnz .LBB5_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v1, v2
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v3, v17, v16
@@ -561,12 +561,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: global_load_dword v14, v[3:4], off
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: global_load_dword v8, v[0:1], off
; GISEL-NEXT: global_load_dword v9, v[3:4], off
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: v_mov_b32_e32 v13, v8
; GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -588,17 +588,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: v_mov_b32_e32 v10, v8
; GISEL-NEXT: v_mov_b32_e32 v14, v9
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(2, ALL)
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v1, v2
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v3, v17, v16
@@ -617,13 +617,13 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Load second iteration
%v1 = load i32, ptr addrspace(1) %foo
%g1 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %loop_body
@@ -646,10 +646,10 @@ loop_body:
%cur_v = load i32, ptr addrspace(1) %foo
%cur_g = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -662,13 +662,13 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e0 = add i32 %sum, %g1_phi
%sum_e1 = add i32 %v1_phi, %sum_e0
%sum_e2 = add i32 %sum_e1, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e3 = add i32 %cur_v, %cur_g
%sum_e4 = add i32 %sum_e3, %lds_val_n_1
@@ -689,9 +689,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -706,9 +706,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -717,9 +717,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -739,12 +739,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_readfirstlane_b32 s4, v5
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: s_mov_b32 m0, s4
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(ALL)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, ALL)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -761,12 +761,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_readfirstlane_b32 s4, v5
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(ALL)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -776,11 +776,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
new file mode 100644
index 0000000000000..f66a4008586d8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
@@ -0,0 +1,48 @@
+; RUN: split-file %s %t
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
+
+; Invalid asyncmark stages are rejected during instruction selection, with the
+; same wording from both selectors. SIInsertWaitcnts must never see them.
+
+;--- oob.ll
+; Values above ALL, and values between STAGE_LAST and ALL, are out of range.
+; OOB: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+define amdgpu_kernel void @asyncmark_out_of_range() {
+ call void @llvm.amdgcn.asyncmark(i32 11)
+ call void @llvm.amdgcn.asyncmark(i32 17)
+ ret void
+}
+
+;--- oob-wait.ll
+; OOB-WAIT: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+define amdgpu_kernel void @wait_asyncmark_out_of_range() {
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 15)
+ ret void
+}
+
+;--- reserved.ll
+; Reserved stages are rejected too.
+; RESERVED: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+define amdgpu_kernel void @asyncmark_reserved() {
+ call void @llvm.amdgcn.asyncmark(i32 4)
+ call void @llvm.amdgcn.asyncmark(i32 6)
+ call void @llvm.amdgcn.asyncmark(i32 7)
+ call void @llvm.amdgcn.asyncmark(i32 8)
+ call void @llvm.amdgcn.asyncmark(i32 9)
+ call void @llvm.amdgcn.asyncmark(i32 10)
+ ret void
+}
+
+;--- reserved-wait.ll
+; RESERVED-WAIT: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+define amdgpu_kernel void @wait_asyncmark_reserved() {
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 4)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
new file mode 100644
index 0000000000000..6a054369559b3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
@@ -0,0 +1,218 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# The MaxAsyncMarks cap (16) is applied per stage, since each stage owns its own
+# mark sequence. Saturating one stage must not perturb another.
+
+---
+# Stage 1 receives 20 marks, well past the cap, while stage 0 receives exactly
+# one. The wait on stage 0 must still find its single mark and produce an exact
+# tensorcnt wait: it is not affected by stage 1's truncation.
+name: saturate_one_stage_only
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ ; CHECK-LABEL: name: saturate_one_stage_only
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 0
+ ; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+
+ ; Stage 0 has one mark; counting back 0 reaches it exactly.
+ WAIT_ASYNCMARK 0, 0
+ S_ENDPGM 0
+...
+---
+# Waiting deep into a saturated stage. N is clamped to MaxAsyncMarks-1 so that a
+# non-trivial wait is still generated after truncation.
+name: wait_deep_in_saturated_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ ; CHECK-LABEL: name: wait_deep_in_saturated_stage
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 18, 1
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 18, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+
+ WAIT_ASYNCMARK 18, 1
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
new file mode 100644
index 0000000000000..c5f20d94e14ba
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
@@ -0,0 +1,284 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# Merging asyncmark brackets at a CFG join, with marks in different stages.
+#
+# Each stage keeps an independent mark sequence, so the join must merge each
+# sequence against its counterpart in the other predecessor and no other. The
+# zero-mark identity front-padding that equalises differing sequence lengths
+# has to be applied per stage rather than once across all of them.
+
+---
+# Both predecessors mark, but in different stages. Neither stage's sequence has
+# a counterpart in the other predecessor, so both survive the join as
+# single-element sequences padded with the zero-mark identity.
+#
+# The join then waits on stage 1 only. The tensor DMA from bb.2 must stay in
+# flight: no S_WAIT_TENSORCNT may appear.
+name: merge_disjoint_stages
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_disjoint_stages
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) on ASYNC_CNT.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ ; Stage 0 (TENSOR) on TENSOR_CNT.
+ bb.2:
+ liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 0
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 1
+ S_ENDPGM 0
+...
+---
+# One predecessor is empty in the stage being waited on. Merging a one-element
+# sequence against an empty one must not disturb the other stage's sequence,
+# which is present in both predecessors.
+name: merge_one_side_empty_in_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_one_side_empty_in_stage
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Marks in both stage 0 and stage 1.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 0
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ ; Marks in stage 0 only; stage 1's sequence is empty on this path.
+ bb.2:
+ liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+ TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
+ ASYNCMARK 0
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 0
+ WAIT_ASYNCMARK 0, 1
+ S_ENDPGM 0
+...
+---
+# Sequences of differing length in the same stage. The shorter one is padded at
+# the front with zero marks, so counting back N marks from the join reaches the
+# conservative (most recent) mark on both paths.
+name: merge_unequal_lengths_same_stage
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_unequal_lengths_same_stage
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; Two marks in stage 1.
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ ; One mark in stage 1.
+ bb.2:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 1
+ S_ENDPGM 0
+...
+---
+# A stage-specific sequence and the ALL sequence merged at the same join. ALL is
+# held separately from the dense stage array, so it must take part in the merge
+# on the same footing.
+name: merge_stage_and_all
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: merge_stage_and_all
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 16
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ bb.1:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 16
+ S_BRANCH %bb.3
+
+ bb.2:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK 1
+ S_BRANCH %bb.3
+
+ bb.3:
+ WAIT_ASYNCMARK 0, 16
+ WAIT_ASYNCMARK 0, 1
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
new file mode 100644
index 0000000000000..2a988b4426eab
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=GISEL
+
+; Stage UNFORMATTED_BUFFER_GLOBAL_LOAD maps to LOAD_CNT, which pre-gfx12 is
+; vmcnt. Buffer async LDS loads are the operations in that stage.
+
+; Marks in stage 5 form their own sequence; wait_asyncmark(1, 5) counts back
+; through it and produces a vmcnt wait, not a blanket wait.
+
+define amdgpu_kernel void @stage_unformatted_buffer_load(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: stage_unformatted_buffer_load:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_mov_b32 m0, s6
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; SDAG-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
+; SDAG-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
+; SDAG-NEXT: v_mov_b32_e32 v0, s6
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; SDAG-NEXT: ; wait_asyncmark(1, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: s_waitcnt vmcnt(2)
+; SDAG-NEXT: ds_read_b32 v0, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v1, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: stage_unformatted_buffer_load:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_mov_b32 m0, s6
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; GISEL-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
+; GISEL-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
+; GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; GISEL-NEXT: ; wait_asyncmark(1, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: s_waitcnt vmcnt(2)
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 5)
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 5)
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 5)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; A wait on a stage that never received a mark is a no-op, even though the
+; outstanding buffer loads are on the counter that stage would map to.
+
+define amdgpu_kernel void @wait_on_empty_stage(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: wait_on_empty_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_mov_b32 m0, s6
+; SDAG-NEXT: v_mov_b32_e32 v0, s6
+; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ds_read_b32 v0, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v1, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_on_empty_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_mov_b32 m0, s6
+; GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 5)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
+declare void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32>, ptr addrspace(3), i32, i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
new file mode 100644
index 0000000000000..bf4b2e3927463
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
@@ -0,0 +1,400 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GISEL
+
+; Asyncmark stages partition the mark sequence. A wait on one stage must not
+; observe counters belonging to another stage, and must not count back through
+; marks that belong to another stage even when the two stages share a counter.
+
+; The over-waiting fix. Tensor DMA (TENSOR_CNT) and async global-to-LDS
+; (ASYNC_CNT) are interleaved, each with its own marks. wait_asyncmark(0, TENSOR)
+; must wait only on tensorcnt: the outstanding async load is unrelated to it and
+; must stay in flight.
+
+define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_tensor_ignores_async:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(TENSOR)
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, TENSOR)
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_tensor_ignores_async:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(TENSOR)
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, TENSOR)
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v0, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ ; Waits for the tensor DMA only. No s_wait_asynccnt may be emitted here.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; Same shape, but waiting on the async stage instead. Now asynccnt is required
+; and tensorcnt is not.
+
+define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_async_ignores_tensor:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(TENSOR)
+; SDAG-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_async_ignores_tensor:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(TENSOR)
+; GISEL-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: ds_load_b32 v1, v1
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v1, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; Two stages that share ASYNC_CNT. Stages are still independent mark sequences:
+; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS) counts back through the loads only,
+; so the intervening store mark does not shift the wait. Both loads and the store
+; are on asynccnt, so a wait is emitted, but its value reflects one sequence.
+
+define amdgpu_kernel void @same_counter_distinct_sequences(ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: same_counter_distinct_sequences:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_add_co_i32 s3, s2, 4
+; SDAG-NEXT: s_add_co_i32 s2, s2, 8
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(ASYNC_LDS_STORE)
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(1, GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: s_wait_asynccnt 0x2
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: same_counter_distinct_sequences:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GISEL-NEXT: s_add_co_u32 s3, s2, 4
+; GISEL-NEXT: s_add_co_u32 s2, s2, 8
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(ASYNC_LDS_STORE)
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(1, GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: s_wait_asynccnt 0x2
+; GISEL-NEXT: ds_load_b32 v1, v1
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ %lds2 = getelementptr i32, ptr addrspace(3) %lds, i32 2
+
+ ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) mark #0.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ ; Stage 3 (ASYNC_LDS_STORE) mark #0. Same counter, different sequence.
+ call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 3)
+
+ ; Stage 1 mark #1.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds2, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ ; Counts back one mark in stage 1's sequence, skipping the stage 3 mark.
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 1)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; Stage ALL is the pre-stage behavior: a mark observes every counter and a wait
+; on it waits for all of them. Marks placed in ALL are their own sequence, so
+; stage-specific marks do not perturb it.
+
+define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: stage_all_waits_for_everything:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: stage_all_waits_for_everything:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v0, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A wait on a stage with no marks at all is a no-op: nothing to count back to.
+
+define amdgpu_kernel void @wait_on_empty_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: wait_on_empty_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, TENSOR)
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: wait_on_empty_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, TENSOR)
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ ; Stage 0 (TENSOR) never had a mark, so this waits for nothing.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; The multicast (cluster) loads have their own stage, distinct from the plain
+; async loads to LDS even though both use the async counter. A wait on the
+; multicast stage must not count back through a plain load's mark.
+define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 %mask) {
+; SDAG-LABEL: mcast_is_a_distinct_stage:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_add_co_i32 s2, s2, 4
+; SDAG-NEXT: s_mov_b32 m0, s3
+; SDAG-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
+; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v1 offset:4
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: mcast_is_a_distinct_stage:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GISEL-NEXT: s_add_co_u32 s2, s2, 4
+; GISEL-NEXT: s_mov_b32 m0, s3
+; GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
+; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: ds_load_b32 v1, v1 offset:4
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+
+ ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) mark.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 1)
+
+ ; Stage 2 (GLOBAL_LOAD_ASYNC_TO_LDS_MCAST) mark. Same counter, own sequence.
+ call void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0, i32 %mask)
+ call void @llvm.amdgcn.asyncmark(i32 2)
+
+ ; Drains stage 2 only. The stage 1 mark is in a different sequence.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2)
+ %v = load i32, ptr addrspace(3) %lds1
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
+declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32>, <8 x i32>, <4 x i32>, <4 x i32>, <8 x i32>, i32)
+declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32)
+declare void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32)
+declare void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1), ptr addrspace(3), i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
index 683e2ba0af75b..015e413238a1d 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-waitcnt.mir
@@ -17,9 +17,9 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
- ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK 16
; CHECK-NEXT: S_ENDPGM 0
S_WAITCNT_soft 0
- ASYNCMARK
+ ASYNCMARK 16
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
index 681b584d3e48b..035544d34bd40 100644
--- a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
+++ b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
@@ -35,7 +35,7 @@ body: |
# WAIT_ASYNCMARK is a meta instruction that emits zero bytes.
# codeLenInByte = s_waitcnt (4) + WAIT_ASYNCMARK (0) = 4.
# CHECK: wait_asyncmark_meta: ; @wait_asyncmark_meta
-# CHECK: ; wait_asyncmark(1)
+# CHECK: ; wait_asyncmark(1, ALL)
# CHECK: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
# CHECK: ; codeLenInByte = 4
---
@@ -44,7 +44,7 @@ tracksRegLiveness: true
body: |
bb.0:
- WAIT_ASYNCMARK 1
+ WAIT_ASYNCMARK 1, 16
...
# CHECK: align4: ; @align4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
index 5d85ef0c8239c..a65dacd1f9067 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -5,8 +5,8 @@
; %D4 should be zero-initialized for gfx1250, which only supports 4 groups of tensor descriptor
declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> %D4, i32 %cpol)
declare void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, <8 x i32> %D4, i32 %cpol)
-declare void @llvm.amdgcn.asyncmark()
-declare void @llvm.amdgcn.wait.asyncmark(i16)
+declare void @llvm.amdgcn.asyncmark(i32)
+declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 %offset, i32 %cpol)
define amdgpu_ps void @tensor_load_to_lds_d4(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
@@ -331,13 +331,13 @@ define amdgpu_ps void @tensor_load_to_lds_with_asyncmark(<4 x i32> inreg %D0, <8
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(ALL)
+; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
ret void
}
@@ -349,13 +349,13 @@ define amdgpu_ps void @tensor_store_from_lds_with_asyncmark(<4 x i32> inreg %D0,
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(ALL)
+; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
ret void
}
@@ -370,14 +370,14 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark
+; GFX1250-NEXT: ; asyncmark(ALL)
; GFX1250-NEXT: s_wait_tensorcnt 0xa
; GFX1250-NEXT: tensor_load_to_lds s[12:15], s[16:23]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(1)
+; GFX1250-NEXT: ; asyncmark(ALL)
+; GFX1250-NEXT: ; wait_asyncmark(1, ALL)
; GFX1250-NEXT: s_wait_tensorcnt 0x1
; GFX1250-NEXT: ds_load_b32 v1, v0
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: ds_load_b32 v2, v0 offset:4
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -385,15 +385,15 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: ds_store_b32 v0, v1
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0a, <8 x i32> %D1a, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0b, <8 x i32> %D1b, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
%lds_v0 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%lds_v1 = load i32, ptr addrspace(3) %lds_gep1
@@ -415,15 +415,15 @@ define void @tensor_and_async_lds_with_asyncmark(<4 x i32> inreg %D0, <8 x i32>
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[16:23]
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: ; asyncmark(ALL)
+; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
; GFX1250-NEXT: s_wait_asynccnt 0x0
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
- call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
ret void
}
@@ -455,7 +455,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.1: ; %g1
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-SDAG-NEXT: s_mov_b32 s0, 0
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
; GFX1250-SDAG-NEXT: .LBB14_2: ; %Flow1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
@@ -464,7 +464,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-SDAG-NEXT: ; %bb.3: ; %t1
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
; GFX1250-SDAG-NEXT: .LBB14_4: ; %merge1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
@@ -472,7 +472,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.5: ; %g2
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-SDAG-NEXT: s_mov_b32 s0, 0
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
; GFX1250-SDAG-NEXT: .LBB14_6: ; %Flow
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
@@ -482,9 +482,9 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.7: ; %t2
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0xa
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-SDAG-NEXT: ; asyncmark
+; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
; GFX1250-SDAG-NEXT: .LBB14_8: ; %merge2
-; GFX1250-SDAG-NEXT: ; wait_asyncmark(1)
+; GFX1250-SDAG-NEXT: ; wait_asyncmark(1, ALL)
; GFX1250-SDAG-NEXT: s_wait_asynccnt 0x0
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -511,7 +511,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.1: ; %g1
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
; GFX1250-GISEL-NEXT: .LBB14_2: ; %Flow1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_xor_b32 s0, s0, 1
@@ -519,7 +519,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-GISEL-NEXT: ; %bb.3: ; %t1
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
; GFX1250-GISEL-NEXT: .LBB14_4: ; %merge1
; GFX1250-GISEL-NEXT: s_cmp_eq_u32 s1, 0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 1
@@ -527,7 +527,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.5: ; %g2
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
; GFX1250-GISEL-NEXT: .LBB14_6: ; %Flow
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_xor_b32 s0, s0, 1
@@ -536,9 +536,9 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.7: ; %t2
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0xa
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-GISEL-NEXT: ; asyncmark
+; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
; GFX1250-GISEL-NEXT: .LBB14_8: ; %merge2
-; GFX1250-GISEL-NEXT: ; wait_asyncmark(1)
+; GFX1250-GISEL-NEXT: ; wait_asyncmark(1, ALL)
; GFX1250-GISEL-NEXT: s_wait_asynccnt 0x0
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0x0
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -548,12 +548,12 @@ entry:
t1:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %merge1
g1:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %merge1
merge1:
@@ -562,15 +562,15 @@ merge1:
t2:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %merge2
g2:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark(i32 16)
br label %merge2
merge2:
- call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
ret void
}
>From 576086a4f0f90cda687680c7f72aa0a4066097d1 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Wed, 2 Sep 2026 12:59:34 -0700
Subject: [PATCH 2/4] Use machineverifier and rangeset
---
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 8 +++--
llvm/lib/IR/AutoUpgrade.cpp | 2 +-
.../AMDGPU/AMDGPUInstructionSelector.cpp | 13 +------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 16 ---------
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 10 ++++++
.../CodeGen/AMDGPU/asyncmark-stage-err.ll | 34 +++++++++++--------
.../CodeGen/AMDGPU/asyncmark-stage-err.mir | 30 ++++++++++++++++
7 files changed, 67 insertions(+), 46 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 9ced72f61fe8d..f0465388ba9f9 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2923,14 +2923,18 @@ def int_amdgcn_pops_exiting_wave_id :
// Sets a marker in the stream of async requests. Modelled as InaccessibleMem.
def int_amdgcn_asyncmark : ClangBuiltin<"__builtin_amdgcn_asyncmark">,
- Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
+ Intrinsic<[], [llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, RangeSet<ArgIndex<0>, [[0, 3], [5, 5], [16, 16]]>,
+ IntrNoMem, IntrHasSideEffects]>;
// Waits until the Nth previous marker of the given stage is completed, if it
// exists.
def int_amdgcn_wait_asyncmark :
ClangBuiltin<"__builtin_amdgcn_wait_asyncmark">,
Intrinsic<[], [llvm_i16_ty, llvm_i32_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, IntrNoMem, IntrHasSideEffects]>;
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ RangeSet<ArgIndex<1>, [[0, 3], [5, 5], [16, 16]]>,
+ IntrNoMem, IntrHasSideEffects]>;
//===----------------------------------------------------------------------===//
// GFX10 Intrinsics
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index aa5ea83cad903..980c9fcada541 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5137,7 +5137,7 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
// Legacy asyncmark intrinsics missed the stage operand. Append the ALL stage,
// which is the behavior they had: every async operation belongs to it.
if ((F->getIntrinsicID() == Intrinsic::amdgcn_asyncmark &&
- CI->arg_size() == 0) ||
+ CI->arg_empty()) ||
(F->getIntrinsicID() == Intrinsic::amdgcn_wait_asyncmark &&
CI->arg_size() == 1)) {
SmallVector<Value *, 2> Args(CI->args());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index c7e6ba9fcb1c8..893c7cce1a0f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -26,7 +26,6 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/Support/AMDGPUAsyncStages.h"
#include <optional>
#define DEBUG_TYPE "amdgpu-isel"
@@ -2439,20 +2438,10 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
case Intrinsic::amdgcn_tensor_store_from_lds:
return selectTensorLoadStore(I, IntrinsicID);
case Intrinsic::amdgcn_asyncmark:
- case Intrinsic::amdgcn_wait_asyncmark: {
+ case Intrinsic::amdgcn_wait_asyncmark:
if (!Subtarget->hasAsyncMark())
return false;
- uint32_t Stage = I.getOperand(I.getNumOperands() - 1).getImm();
- if (!AMDGPU::AsyncStage::isValidStage(Stage) ||
- AMDGPU::AsyncStage::isReservedStage(Stage)) {
- const Function &Fn = MF->getFunction();
- Fn.getContext().diagnose(DiagnosticInfoUnsupported(
- Fn, "intrinsic @llvm.amdgcn.*.asyncmark: invalid stage",
- I.getDebugLoc(), DS_Error));
- return false;
- }
break;
- }
case Intrinsic::amdgcn_ds_bvh_stack_rtn:
case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a27b8224f5063..a8b4d1f9a568b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -45,7 +45,6 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
-#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
@@ -12559,21 +12558,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
initializeM0ToZeroForClusterLoad(Op, DAG, DL);
return SDValue();
}
- case Intrinsic::amdgcn_asyncmark:
- case Intrinsic::amdgcn_wait_asyncmark: {
- Function &F = DAG.getMachineFunction().getFunction();
- auto *StageC =
- cast<ConstantSDNode>(Op->getOperand(Op.getNumOperands() - 1));
- uint32_t StageRaw = StageC->getZExtValue();
- if (!AMDGPU::AsyncStage::isValidStage(StageRaw) ||
- AMDGPU::AsyncStage::isReservedStage(StageRaw)) {
- F.getContext().diagnose(DiagnosticInfoUnsupported(
- F, "intrinsic @llvm.amdgcn.*.asyncmark: invalid stage",
- DL.getDebugLoc(), DS_Error));
- return Chain;
- }
- return SDValue();
- }
case Intrinsic::amdgcn_exp_compr: {
SDValue Src0 = Op.getOperand(4);
SDValue Src1 = Op.getOperand(5);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3094c1390e9cf..a4a2da25ed34a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -33,6 +33,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/MC/MCContext.h"
+#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Target/TargetMachine.h"
#include <tuple>
@@ -5755,6 +5756,15 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
}
+ if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
+ int64_t Stage = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
+ if (Stage < 0 || !AMDGPU::AsyncStage::isValidStage(Stage) ||
+ AMDGPU::AsyncStage::isReservedStage(Stage)) {
+ ErrInfo = "invalid asyncmark stage";
+ return false;
+ }
+ }
+
// Verify misc. restrictions on specific instructions.
if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
index f66a4008586d8..0111a251d4415 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
@@ -1,19 +1,17 @@
; RUN: split-file %s %t
-; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
-; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
-; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
-; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
-; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
-; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
-; RUN: not llc -global-isel=0 -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
-; RUN: not llc -global-isel=1 -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
-; Invalid asyncmark stages are rejected during instruction selection, with the
-; same wording from both selectors. SIInsertWaitcnts must never see them.
+; The stage operand carries a RangeSet covering the valid, non-reserved stages,
+; so the IR verifier rejects bad stages before instruction selection. Reserved
+; stages are simply the gaps in that set.
;--- oob.ll
; Values above ALL, and values between STAGE_LAST and ALL, are out of range.
-; OOB: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+; OOB: immarg value 11 for arg 0 out of range set
+; OOB: immarg value 17 for arg 0 out of range set
define amdgpu_kernel void @asyncmark_out_of_range() {
call void @llvm.amdgcn.asyncmark(i32 11)
call void @llvm.amdgcn.asyncmark(i32 17)
@@ -21,15 +19,21 @@ define amdgpu_kernel void @asyncmark_out_of_range() {
}
;--- oob-wait.ll
-; OOB-WAIT: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+; OOB-WAIT: immarg value 15 for arg 1 out of range set
define amdgpu_kernel void @wait_asyncmark_out_of_range() {
call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 15)
ret void
}
;--- reserved.ll
-; Reserved stages are rejected too.
-; RESERVED: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+; Every reserved stage is rejected. These hold slots in the stage numbering for
+; async operations that do not exist yet.
+; RESERVED: immarg value 4 for arg 0 out of range set
+; RESERVED: immarg value 6 for arg 0 out of range set
+; RESERVED: immarg value 7 for arg 0 out of range set
+; RESERVED: immarg value 8 for arg 0 out of range set
+; RESERVED: immarg value 9 for arg 0 out of range set
+; RESERVED: immarg value 10 for arg 0 out of range set
define amdgpu_kernel void @asyncmark_reserved() {
call void @llvm.amdgcn.asyncmark(i32 4)
call void @llvm.amdgcn.asyncmark(i32 6)
@@ -41,7 +45,7 @@ define amdgpu_kernel void @asyncmark_reserved() {
}
;--- reserved-wait.ll
-; RESERVED-WAIT: error: {{.*}}intrinsic @llvm.amdgcn.*.asyncmark: invalid stage
+; RESERVED-WAIT: immarg value 4 for arg 1 out of range set
define amdgpu_kernel void @wait_asyncmark_reserved() {
call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
new file mode 100644
index 0000000000000..05d8c7e9c00a1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
@@ -0,0 +1,30 @@
+# RUN: split-file %s %t
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none -verify-machineinstrs %t/mark.mir -o /dev/null 2>&1 | FileCheck %s -check-prefix=MARK
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none -verify-machineinstrs %t/wait.mir -o /dev/null 2>&1 | FileCheck %s -check-prefix=WAIT
+
+#--- mark.mir
+# A reserved stage is as invalid as an out-of-range one.
+# MARK: *** Bad machine code: invalid asyncmark stage ***
+# MARK-NEXT: - function: asyncmark_reserved_stage
+# MARK: ASYNCMARK 4
+---
+name: asyncmark_reserved_stage
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ASYNCMARK 4
+ S_ENDPGM 0
+...
+
+#--- wait.mir
+# WAIT: *** Bad machine code: invalid asyncmark stage ***
+# WAIT-NEXT: - function: wait_asyncmark_out_of_range
+# WAIT: WAIT_ASYNCMARK 0, 17
+---
+name: wait_asyncmark_out_of_range
+tracksRegLiveness: true
+body: |
+ bb.0:
+ WAIT_ASYNCMARK 0, 17
+ S_ENDPGM 0
+...
>From d1ca2dde6e272c4996c4afae7a6f1422689bdadd Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Thu, 3 Sep 2026 12:36:05 -0700
Subject: [PATCH 3/4] Enum nits
---
llvm/include/llvm/Support/AMDGPUAsyncStages.h | 20 +++++++-------
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 26 +++++++------------
2 files changed, 19 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/Support/AMDGPUAsyncStages.h b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
index 22a3f59b4317a..11ca455927c7d 100644
--- a/llvm/include/llvm/Support/AMDGPUAsyncStages.h
+++ b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
@@ -15,6 +15,7 @@
#define LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
#include <cstdint>
+#include "llvm/Support/ErrorHandling.h"
namespace llvm {
namespace AMDGPU {
@@ -42,15 +43,9 @@ enum Stage : uint32_t {
STAGE_LAST = RESERVED_10,
ALL = 16,
+ NUM_STAGE_SLOTS = ALL + 1
};
-/// Number of distinct non-ALL stages. ALL is not part of this range.
-constexpr unsigned NumStages = STAGE_LAST + 1;
-
-/// Number of slots needed to index every stage, including ALL. The
-/// slots between STAGE_LAST and ALL are unused.
-constexpr unsigned NumStageSlots = ALL + 1;
-
constexpr bool isValidStage(uint32_t S) { return S <= STAGE_LAST || S == ALL; }
constexpr bool isReservedStage(uint32_t S) {
@@ -62,9 +57,15 @@ constexpr bool isReservedStage(uint32_t S) {
case RESERVED_9:
case RESERVED_10:
return true;
- default:
+ case TENSOR:
+ case GLOBAL_LOAD_ASYNC_TO_LDS:
+ case GLOBAL_LOAD_ASYNC_TO_LDS_MCAST:
+ case ASYNC_LDS_STORE:
+ case UNFORMATTED_BUFFER_GLOBAL_LOAD:
+ case ALL:
return false;
}
+ llvm_unreachable("Unhandled stage");
}
constexpr const char *getStageName(uint32_t S) {
@@ -93,9 +94,8 @@ constexpr const char *getStageName(uint32_t S) {
return "RESERVED_10";
case ALL:
return "ALL";
- default:
- return "invalid";
}
+ llvm_unreachable("Unhandled stage");
}
} // namespace AsyncStage
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 40d894988355e..8c8b4079a8919 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -134,14 +134,6 @@ static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
}
}
-/// Interpret an asyncmark stage operand.
-static AMDGPU::AsyncStage::Stage getAsyncStage(int64_t Imm) {
- if (Imm < 0 || !AMDGPU::AsyncStage::isValidStage(Imm) ||
- AMDGPU::AsyncStage::isReservedStage(Imm))
- return AMDGPU::AsyncStage::ALL;
- return static_cast<AMDGPU::AsyncStage::Stage>(Imm);
-}
-
static bool updateVMCntOnly(const MachineInstr &Inst) {
return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
SIInstrInfo::isFLATGlobal(Inst) || SIInstrInfo::isFLATScratch(Inst);
@@ -423,7 +415,7 @@ class SIInsertWaitcnts {
}
std::optional<AMDGPU::AsyncStage::Stage>
- shouldUpdateAsyncMark(const MachineInstr &MI,
+ getAsyncStageToUpdate(const MachineInstr &MI,
AMDGPU::InstCounterType T) const {
if (SIInstrInfo::usesTENSOR_CNT(MI) && T == AMDGPU::TENSOR_CNT)
return AMDGPU::AsyncStage::TENSOR;
@@ -840,7 +832,7 @@ class WaitcntBrackets {
// State of all counters at each async mark encountered so far, per stage.
std::array<SmallVector<CounterValueArray, 0>,
- AMDGPU::AsyncStage::NumStageSlots>
+ AMDGPU::AsyncStage::NUM_STAGE_SLOTS>
AsyncMarks;
// But in the rare pathological case, a nest of loops that pushes marks
@@ -851,7 +843,7 @@ class WaitcntBrackets {
// Track the upper bound score for async operations that are not part of a
// mark yet, per stage. Initialized to all zeros.
- std::array<CounterValueArray, AMDGPU::AsyncStage::NumStageSlots> AsyncScore{};
+ std::array<CounterValueArray, AMDGPU::AsyncStage::NUM_STAGE_SLOTS> AsyncScore{};
};
SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
@@ -1120,7 +1112,7 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
setVMemScore(LDSDMA_BEGIN + Slot, T, CurrScore);
}
- if (auto Stage = Context->shouldUpdateAsyncMark(Inst, T)) {
+ if (auto Stage = Context->getAsyncStageToUpdate(Inst, T)) {
AsyncScore[AMDGPU::AsyncStage::ALL][T] = CurrScore;
AsyncScore[*Stage][T] = CurrScore;
}
@@ -1251,7 +1243,7 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
}
OS << '\n';
- for (unsigned S = 0; S < AMDGPU::AsyncStage::NumStageSlots; ++S) {
+ for (unsigned S = 0; S < AMDGPU::AsyncStage::NUM_STAGE_SLOTS; ++S) {
if (!AMDGPU::AsyncStage::isValidStage(S))
continue;
OS << "Async score (stage " << AMDGPU::AsyncStage::getStageName(S) << "): ";
@@ -1755,7 +1747,7 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
unsigned N = II.getOperand(0).getImm();
LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, getAsyncStage(II.getOperand(1).getImm()));
+ N, static_cast<AMDGPU::AsyncStage::Stage>(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
@@ -2047,7 +2039,7 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
// shows up in the assembly as a comment with the original parameter N.
unsigned N = II.getOperand(0).getImm();
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, getAsyncStage(II.getOperand(1).getImm()));
+ N, static_cast<AMDGPU::AsyncStage::Stage>(II.getOperand(1).getImm()));
Wait = Wait.combined(OldWait);
} else {
std::optional<AMDGPU::InstCounterType> CT =
@@ -2975,7 +2967,7 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
- for (unsigned S = 0; S != AMDGPU::AsyncStage::NumStageSlots; ++S) {
+ for (unsigned S = 0; S != AMDGPU::AsyncStage::NUM_STAGE_SLOTS; ++S) {
if (!AMDGPU::AsyncStage::isValidStage(S))
continue;
auto Stage = static_cast<AMDGPU::AsyncStage::Stage>(S);
@@ -3162,7 +3154,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// waitcnts that occur after them to be merged into waitcnts that occur
// before.
ScoreBrackets.recordAsyncMark(Inst,
- getAsyncStage(Inst.getOperand(0).getImm()));
+ static_cast<AMDGPU::AsyncStage::Stage>(Inst.getOperand(0).getImm()));
continue;
}
>From 967f296d915d95213d08de110b553dab06374132 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Tue, 8 Sep 2026 19:00:31 -0700
Subject: [PATCH 4/4] Bitmask implementation
---
clang/include/clang/Basic/BuiltinsAMDGPU.td | 4 +-
.../clang/Basic/DiagnosticSemaKinds.td | 3 -
clang/lib/Sema/SemaAMDGPU.cpp | 27 +-
.../builtins-amdgcn-asyncmark-errs-gfx1250.cl | 41 ++-
.../builtins-amdgcn-asyncmark-errs.cl | 4 +-
.../builtins-amdgcn-asyncmark.cl | 50 +--
llvm/docs/AMDGPUAsyncOperations.md | 126 +++++---
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 9 +-
llvm/include/llvm/Support/AMDGPUAsyncStages.h | 49 ++-
llvm/lib/IR/AutoUpgrade.cpp | 7 +-
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 20 +-
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 155 ++++-----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +-
llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll | 21 +-
.../test/CodeGen/AMDGPU/async-buffer-loads.ll | 52 +--
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 138 ++++----
.../CodeGen/AMDGPU/asyncmark-max-pregfx12.ll | 167 +++++-----
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 192 +++++------
.../CodeGen/AMDGPU/asyncmark-stage-err.ll | 66 ++--
.../CodeGen/AMDGPU/asyncmark-stage-err.mir | 30 --
.../CodeGen/AMDGPU/asyncmark-stage-max.mir | 181 +++++------
.../CodeGen/AMDGPU/asyncmark-stage-merge.mir | 91 +++---
.../AMDGPU/asyncmark-stage-pregfx12.ll | 36 +--
llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll | 304 +++++++++++++-----
.../CodeGen/AMDGPU/code-size-estimate.mir | 4 +-
.../AMDGPU/llvm.amdgcn.tensor.load.store.ll | 70 ++--
.../AMDGPU/asyncmark-stage-err.mir | 29 ++
27 files changed, 1036 insertions(+), 847 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
create mode 100644 llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index a6682d69e8350..e4e20a336b69f 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -337,12 +337,12 @@ def __builtin_amdgcn_av_store_b128
def __builtin_amdgcn_asyncmark
: AMDGPUBuiltin<"void(_Constant unsigned int)", [],
"vmem-to-lds-load-insts|asynccnt"> {
- let ArgNames = ["stage"];
+ let ArgNames = ["stage_mask"];
}
def __builtin_amdgcn_wait_asyncmark
: AMDGPUBuiltin<"void(_Constant unsigned short, _Constant unsigned int)", [],
"vmem-to-lds-load-insts|asynccnt"> {
- let ArgNames = ["num_marks", "stage"];
+ let ArgNames = ["num_marks", "stage_mask"];
}
//===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 07510286db89c..24ecc88d2fbc0 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -14391,9 +14391,6 @@ def err_amdgcn_dmask_has_too_many_bits_set
: Error<"dmask argument cannot have more bits set than there are elements "
"in return type">;
-def err_amdgcn_asyncmark_stage_reserved
- : Error<"asyncmark stage %0 is reserved">;
-
def warn_amdgpu_s_wait_event_mask_no_effect_target :
Warning<"event mask has no effect for target">,
InGroup<DiagGroup<"amdgpu-wait-event-mask">>;
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index b31085557e4a6..235e284a5b02c 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -213,28 +213,11 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID,
if (!IsMark && SemaRef.BuiltinConstantArg(TheCall, 0, NumMarks))
return true;
- unsigned StageArgNum = IsMark ? 0 : 1;
- llvm::APSInt Stage;
- if (SemaRef.BuiltinConstantArg(TheCall, StageArgNum, Stage))
- return true;
-
- Expr *ArgExpr = TheCall->getArg(StageArgNum);
- uint64_t Value = Stage.getZExtValue();
-
- if (!llvm::AMDGPU::AsyncStage::isValidStage(Value)) {
- return Diag(ArgExpr->getBeginLoc(), diag::err_argument_invalid_range)
- << toString(Stage, 10) << 0 << (int)llvm::AMDGPU::AsyncStage::ALL
- << ArgExpr->getSourceRange();
- }
-
- if (llvm::AMDGPU::AsyncStage::isReservedStage(Value)) {
- return Diag(ArgExpr->getBeginLoc(),
- diag::err_amdgcn_asyncmark_stage_reserved)
- << llvm::AMDGPU::AsyncStage::getStageName(Value)
- << ArgExpr->getSourceRange();
- }
-
- return false;
+ // The stage mask names the stages to leave out, so every combination of
+ // known stage bits is meaningful, including none of them.
+ unsigned MaskArgNum = IsMark ? 0 : 1;
+ return SemaRef.BuiltinConstantArgRange(
+ TheCall, MaskArgNum, 0, llvm::AMDGPU::AsyncStage::MaskAllStages);
}
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
index 7889c1ef04700..4becaa488f5f2 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs-gfx1250.cl
@@ -1,33 +1,30 @@
// REQUIRES: amdgpu-registered-target
// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.50-amd-amdhsa -verify -S -o - %s
-// The stage must be a compile-time constant; _Constant alone does not give a
-// range, so the value is checked here rather than left to the backend.
+// The stage mask must be a compile-time constant.
-void test_stage_not_constant(unsigned int s) {
- __builtin_amdgcn_asyncmark(s); // expected-error{{argument to '__builtin_amdgcn_asyncmark' must be a constant integer}}
- __builtin_amdgcn_wait_asyncmark(0, s); // expected-error{{argument to '__builtin_amdgcn_wait_asyncmark' must be a constant integer}}
+void test_mask_not_constant(unsigned int m) {
+ __builtin_amdgcn_asyncmark(m); // expected-error{{argument to '__builtin_amdgcn_asyncmark' must be a constant integer}}
+ __builtin_amdgcn_wait_asyncmark(0, m); // expected-error{{argument to '__builtin_amdgcn_wait_asyncmark' must be a constant integer}}
}
-// Stages above the catch-all, and the gap between the last named stage and the
-// catch-all, are out of range.
+// A mask may only name stages that exist. 2047 names all eleven of them, and
+// 2048 is the bit just past the last one.
-void test_stage_out_of_range() {
- __builtin_amdgcn_asyncmark(11); // expected-error{{argument value 11 is outside the valid range [0, 16]}}
- __builtin_amdgcn_asyncmark(15); // expected-error{{argument value 15 is outside the valid range [0, 16]}}
- __builtin_amdgcn_asyncmark(17); // expected-error{{argument value 17 is outside the valid range [0, 16]}}
- __builtin_amdgcn_wait_asyncmark(0, 17); // expected-error{{argument value 17 is outside the valid range [0, 16]}}
+void test_mask_out_of_range() {
+ __builtin_amdgcn_asyncmark(2048); // expected-error{{argument value 2048 is outside the valid range [0, 2047]}}
+ __builtin_amdgcn_asyncmark(4096); // expected-error{{argument value 4096 is outside the valid range [0, 2047]}}
+ __builtin_amdgcn_wait_asyncmark(0, 2048); // expected-error{{argument value 2048 is outside the valid range [0, 2047]}}
}
-// The reserved stages hold slots in the taxonomy for asynchronous operations
-// this target does not have. Using one is an error, not a silent no-op.
+// Every combination of in-range bits is accepted, including bits of the
+// reserved stages: leaving out a stage whose operations do not exist yet is
+// harmless, and keeps masks portable as the reserved slots are filled in.
-void test_stage_reserved() {
- __builtin_amdgcn_asyncmark(4); // expected-error{{asyncmark stage RESERVED_4 is reserved}}
- __builtin_amdgcn_asyncmark(6); // expected-error{{asyncmark stage RESERVED_6 is reserved}}
- __builtin_amdgcn_asyncmark(7); // expected-error{{asyncmark stage RESERVED_7 is reserved}}
- __builtin_amdgcn_asyncmark(8); // expected-error{{asyncmark stage RESERVED_8 is reserved}}
- __builtin_amdgcn_asyncmark(9); // expected-error{{asyncmark stage RESERVED_9 is reserved}}
- __builtin_amdgcn_asyncmark(10); // expected-error{{asyncmark stage RESERVED_10 is reserved}}
- __builtin_amdgcn_wait_asyncmark(0, 4); // expected-error{{asyncmark stage RESERVED_4 is reserved}}
+void test_mask_reserved_bits_ok() {
+ __builtin_amdgcn_asyncmark(16); // RESERVED_4
+ __builtin_amdgcn_asyncmark(2000); // every reserved stage at once
+ __builtin_amdgcn_asyncmark(2047); // every stage
+ __builtin_amdgcn_wait_asyncmark(0, 16);
+ __builtin_amdgcn_wait_asyncmark(0, 2047);
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
index 59ee951d98790..55f967b9bdb6e 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark-errs.cl
@@ -3,6 +3,6 @@
// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgpu12.00-amd-amdhsa -verify -S -o - %s
void test_feature() {
- __builtin_amdgcn_asyncmark(16); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
- __builtin_amdgcn_wait_asyncmark(0, 16); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_asyncmark(0); // expected-error{{'__builtin_amdgcn_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
+ __builtin_amdgcn_wait_asyncmark(0, 0); // expected-error{{'__builtin_amdgcn_wait_asyncmark' needs target feature vmem-to-lds-load-insts|asynccnt}}
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
index 69750046e2e9e..2631d6a89cfd1 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-asyncmark.cl
@@ -5,40 +5,42 @@
// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -triple amdgpu12.50-unknown-unknown -emit-llvm -o - %s | FileCheck %s
// REQUIRES: amdgpu-registered-target
-// The stage argument selects which sequence of marks the builtin acts on, and
-// reaches the intrinsic unchanged. Stage 16 is the catch-all that observes
-// every counter.
+// The argument is a mask of the stages to leave out, and reaches the intrinsic
+// unchanged. An empty mask leaves out nothing, so it covers every stage.
// CHECK-LABEL: @test_invocation(
// CHECK-NEXT: entry:
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 16)
-// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
// CHECK-NEXT: ret void
//
void test_invocation() {
- __builtin_amdgcn_asyncmark(16);
- __builtin_amdgcn_wait_asyncmark(0, 16);
+ __builtin_amdgcn_asyncmark(0);
+ __builtin_amdgcn_wait_asyncmark(0, 0);
}
-// Every supported stage is accepted.
+// Every combination of stage bits is meaningful, so the masks below are just
+// the complement of the stages they cover: 2046 leaves in TENSOR alone, 2045
+// GLOBAL_LOAD_ASYNC_TO_LDS alone, and 2038 the two of TENSOR and
+// ASYNC_LDS_STORE. A mask may also name reserved stages, or every stage at once.
-// CHECK-LABEL: @test_stages(
+// CHECK-LABEL: @test_masks(
// CHECK-NEXT: entry:
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 1)
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2)
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 3)
-// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 5)
-// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
-// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 5)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2046)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2045)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2038)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2000)
+// CHECK-NEXT: call void @llvm.amdgcn.asyncmark(i32 2047)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2046)
+// CHECK-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 2038)
// CHECK-NEXT: ret void
//
-void test_stages() {
- __builtin_amdgcn_asyncmark(0);
- __builtin_amdgcn_asyncmark(1);
- __builtin_amdgcn_asyncmark(2);
- __builtin_amdgcn_asyncmark(3);
- __builtin_amdgcn_asyncmark(5);
- __builtin_amdgcn_wait_asyncmark(1, 0);
- __builtin_amdgcn_wait_asyncmark(2, 5);
+void test_masks() {
+ __builtin_amdgcn_asyncmark(2046);
+ __builtin_amdgcn_asyncmark(2045);
+ __builtin_amdgcn_asyncmark(2038);
+ __builtin_amdgcn_asyncmark(2000);
+ __builtin_amdgcn_asyncmark(2047);
+ __builtin_amdgcn_wait_asyncmark(1, 2046);
+ __builtin_amdgcn_wait_asyncmark(2, 2038);
}
diff --git a/llvm/docs/AMDGPUAsyncOperations.md b/llvm/docs/AMDGPUAsyncOperations.md
index 8154e5b443cd7..7c3b6c639036c 100644
--- a/llvm/docs/AMDGPUAsyncOperations.md
+++ b/llvm/docs/AMDGPUAsyncOperations.md
@@ -18,31 +18,37 @@ initiated by that thread.
### Stages
-Every asyncmark belongs to a *stage*, which names a kind of async operation.
-Each async operation has an *own stage*, determined by the instruction that
-initiates it, and *belongs to* that stage and to the stage `ALL`.
+A *stage* names a kind of async operation. Each async operation *belongs to* the
+one stage determined by the instruction that initiates it.
The stages are:
-| Value | Stage | Async operations |
+| Bit | Stage | Async operations |
|---|---|---|
| 0 | `TENSOR` | tensor loads and stores |
| 1 | `GLOBAL_LOAD_ASYNC_TO_LDS` | global loads async to LDS |
| 2 | `GLOBAL_LOAD_ASYNC_TO_LDS_MCAST` | multicast (cluster) global loads async to LDS |
| 3 | `ASYNC_LDS_STORE` | async stores from LDS |
| 5 | `UNFORMATTED_BUFFER_GLOBAL_LOAD` | unformatted buffer and global loads to LDS |
-| 16 | `ALL` | all of the above |
-The values not listed above are reserved for future async operations.
-Using one is an error.
+Bits 4 and 6 through 10 are reserved for future async operations, and no
+operation belongs to them yet.
Which async operations a given subtarget actually has is described in
-{ref}`AMDGPU DMA Operations <amdgpu-dma-operations>`. A stage is valid on every
+{ref}`AMDGPU DMA Operations <amdgpu-dma-operations>`. A stage exists on every
subtarget that supports asyncmarks, whether or not that subtarget has any
operation belonging to it; marking an empty stage is simply a no-op.
-Stage `ALL` is the catch-all: since every async operation belongs to it, an
-asyncmark in `ALL` tracks all async operations whatever their own stage.
+### Stage Masks
+
+Both intrinsics take a *stage mask*: an 11-bit value in which a set bit means
+"do not participate". An asyncmark leaves out the stages its mask names, and a
+wait disregards them. The mask `0` therefore names no stage and so covers all of
+them.
+
+A mask may set the bit of a reserved stage. Leaving out a stage whose operations
+do not exist yet is harmless, and lets a mask keep its meaning as the reserved
+bits are filled in. Setting a bit above 10 is an error.
### Current Sequence
@@ -55,22 +61,26 @@ the function is called the *current sequence of* `S`.
The sequences of distinct stages are independent: appending to one does not
affect the length or contents of any other.
-### `@llvm.amdgcn.asyncmark(i32 %S)`
+### `@llvm.amdgcn.asyncmark(i32 %K)`
-Produces an asyncmark in stage `S` and appends it to the current sequence of
-`S`. `S` must be a constant naming a stage that is not reserved.
+Produces an asyncmark in every stage that the mask `K` does not name, and
+appends it to the current sequence of each. The sequences of the stages named by
+`K` are unaffected. `K` must be a constant stage mask.
-### `@llvm.amdgcn.wait.asyncmark(i16 %N, i32 %S)`
+Each sequence receives its own asyncmark, and those asyncmarks are then removed
+independently by later waits.
-Ensures that the length of the current sequence of `S` is at most `N` by
-removing asyncmarks from the start of that sequence if it is more than `N`. The
-sequences of other stages are unaffected. `S` must be a constant naming a stage
-that is not reserved.
+### `@llvm.amdgcn.wait.asyncmark(i16 %N, i32 %K)`
+
+For every stage that the mask `K` does not name, ensures that the length of the
+current sequence of that stage is at most `N` by removing asyncmarks from the
+start of that sequence if it is more than `N`. The sequences of the stages named
+by `K` are unaffected. `K` must be a constant stage mask.
### Completion of Asyncmarks
An `asyncmark()` operation `X` that produces an asyncmark `M` in stage `S` is
-*completed-at* a `wait.asyncmark()` operation `Y` on stage `S` in the same
+*completed-at* a `wait.asyncmark()` operation `Y` covering `S` in the same
function body if:
- `X` is *program-ordered* before `Y`, and
@@ -91,8 +101,9 @@ in a stage `S` such that:
- `I` is *program-ordered* before `X`, and
- `X` is *completed-at* `Y`.
-Since `A` belongs to `ALL` as well as to its own stage, an asyncmark and wait
-pair on `ALL` tracks `A` regardless of its own stage.
+Since an asyncmark with an empty mask is produced in every stage, an asyncmark
+and wait pair that both use the mask `0` tracks `A` whatever stage it belongs
+to.
### happens-before
@@ -104,10 +115,10 @@ If `A` is *completed-at* a `wait.asyncmark()` operation `Y`, then `A`
## Examples
-The stage argument is omitted in the examples below; they all use stage `ALL`,
+The mask argument is omitted in the examples below; they all use the empty mask,
so every asyncmark tracks every async operation and there is only one sequence
to reason about. The {ref}`interleaved stages <amdgpu-async-interleaved-stages>`
-example shows what stages add.
+example shows what narrower masks add.
### Uneven blocks of async operations
@@ -198,48 +209,72 @@ void foo(global int *g, local int *l) {
(amdgpu-async-interleaved-stages)=
### Interleaved stages
-Two kinds of async operation are in flight at once. Each is marked in its own
-stage, so each can be waited for without waiting for the other.
+Two kinds of async operation are in flight at once. Each is marked with a mask
+that leaves out the other, so each can be waited for without waiting for the
+other.
+
+The masks below are written as `only(...)`, meaning the mask that names every
+stage except the ones listed, so that only those are left in.
```c++
void foo(global int *g, local int *l, tensor_desc t) {
- // Start a long tensor load and mark it in the TENSOR stage.
+ // Start a long tensor load and mark it in the TENSOR stage alone.
tensor_load_to_lds(l, t);
- asyncmark(TENSOR);
+ asyncmark(only(TENSOR));
- // Start a short LDS load and mark it in its own stage.
+ // Start a short LDS load and mark it in its own stage alone.
async_load_to_lds(l, g);
- asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS);
+ asyncmark(only(GLOBAL_LOAD_ASYNC_TO_LDS));
- // Wait for the LDS load only. The tensor load is still in flight: its
- // asyncmark is in a different sequence, so this wait neither counts nor
- // removes it.
- wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+ // Wait for the LDS load only. The tensor load is still in flight: this wait
+ // leaves out TENSOR, so it neither counts nor removes that asyncmark.
+ wait.asyncmark(0, only(GLOBAL_LOAD_ASYNC_TO_LDS));
// perform synchronization / use the data loaded by async_load_to_lds
// Now wait for the tensor load.
- wait.asyncmark(0, TENSOR);
+ wait.asyncmark(0, only(TENSOR));
}
```
-Had both marks been placed in `ALL`, the first wait would have drained the
-tensor load as well, and the overlap would have been lost.
+Had both marks used the empty mask, the first wait would have drained the tensor
+load as well, and the overlap would have been lost.
-The same applies to two stages tracked by one hardware counter. Here both
-stages use the async counter, but the sequences are still separate:
+The same applies to two stages tracked by one hardware counter. Here both stages
+use the async counter, but the sequences are still separate:
```c++
void foo(global int *g, local int *l) {
async_load_to_lds(l, g);
- asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS); // X
+ asyncmark(only(GLOBAL_LOAD_ASYNC_TO_LDS)); // X
async_store_from_lds(g, l);
- asyncmark(ASYNC_LDS_STORE); // Y
+ asyncmark(only(ASYNC_LDS_STORE)); // Y
// Completes X. Y is in a different sequence and is not completed here, even
// though both stages are tracked by the same counter.
- wait.asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS);
+ wait.asyncmark(0, only(GLOBAL_LOAD_ASYNC_TO_LDS));
+}
+```
+
+A mask need not leave in just one stage. An asyncmark whose mask leaves in
+several is appended to each of their sequences, and a wait whose mask leaves in
+several trims each of them:
+
+```c++
+void foo(global int *g, local int *l, tensor_desc t) {
+ tensor_load_to_lds(l, t);
+ async_load_to_lds(l, g);
+
+ // One asyncmark, appended to both sequences.
+ asyncmark(only(TENSOR, GLOBAL_LOAD_ASYNC_TO_LDS));
+
+ // Removes the copy in GLOBAL_LOAD_ASYNC_TO_LDS. The copy in TENSOR is a
+ // separate asyncmark and stays where it is.
+ wait.asyncmark(0, only(GLOBAL_LOAD_ASYNC_TO_LDS));
+
+ // Removes the copy in TENSOR.
+ wait.asyncmark(0, only(TENSOR));
}
```
@@ -329,8 +364,9 @@ cases. These are just examples and not meant to be an exhaustive list. Each
applies per stage: the sequences are independent, so an asyncmark in one stage
is unaffected by the waits of another.
-1. An `asyncmark` operation in a stage `S` which remains in the current
- sequence of `S` along every path that reaches the function exit.
+1. An `asyncmark` in a stage `S` which remains in the current sequence of `S`
+ along every path that reaches the function exit. An `asyncmark` operation is
+ eliminated entirely only once this holds for every stage it covers.
```c++
void foo() {
@@ -342,8 +378,8 @@ is unaffected by the waits of another.
Here, `X` can be eliminated.
-2. A `wait.asyncmark` on a stage `S` which sees an empty sequence of asyncmarks
- for `S` along every path that reaches it.
+2. A `wait.asyncmark` which sees an empty sequence of asyncmarks along every
+ path that reaches it, for every stage it covers.
```c++
void foo() {
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index f0465388ba9f9..f6952feebfe0b 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2922,18 +2922,19 @@ def int_amdgcn_pops_exiting_wave_id :
DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrHasSideEffects]>;
// Sets a marker in the stream of async requests. Modelled as InaccessibleMem.
+// The argument is a mask of the async stages to omit.
def int_amdgcn_asyncmark : ClangBuiltin<"__builtin_amdgcn_asyncmark">,
Intrinsic<[], [llvm_i32_ty],
- [ImmArg<ArgIndex<0>>, RangeSet<ArgIndex<0>, [[0, 3], [5, 5], [16, 16]]>,
+ [ImmArg<ArgIndex<0>>, Range<ArgIndex<0>, 0, 2048>,
IntrNoMem, IntrHasSideEffects]>;
-// Waits until the Nth previous marker of the given stage is completed, if it
-// exists.
+// Waits until the Nth previous marker is completed in each async stage, if it
+// exists. The second argument is a mask of the stages to ignore.
def int_amdgcn_wait_asyncmark :
ClangBuiltin<"__builtin_amdgcn_wait_asyncmark">,
Intrinsic<[], [llvm_i16_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- RangeSet<ArgIndex<1>, [[0, 3], [5, 5], [16, 16]]>,
+ Range<ArgIndex<1>, 0, 2048>,
IntrNoMem, IntrHasSideEffects]>;
//===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Support/AMDGPUAsyncStages.h b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
index 11ca455927c7d..f879154aa7c9e 100644
--- a/llvm/include/llvm/Support/AMDGPUAsyncStages.h
+++ b/llvm/include/llvm/Support/AMDGPUAsyncStages.h
@@ -14,14 +14,22 @@
#ifndef LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
#define LLVM_SUPPORT_AMDGPUASYNCSTAGES_H
-#include <cstdint>
#include "llvm/Support/ErrorHandling.h"
+#include <cstdint>
+#include <string>
namespace llvm {
namespace AMDGPU {
namespace AsyncStage {
-// Stage selector for the asyncmark / wait_asyncmark intrinsics.
+// Async stages tracked by the asyncmark / wait_asyncmark intrinsics. Each stage
+// has its own independent sequence of marks.
+//
+// The intrinsics do not name a stage directly. They take a bitmask in which a
+// set bit means "do not participate": asyncmark omits the stages it names, and
+// wait_asyncmark ignores them. A mask of 0 therefore covers every stage, which
+// is the behavior of the original stage-less intrinsics.
+//
// Do not renumber. Some values are RESERVED for later use.
enum Stage : uint32_t {
// Tensor loads to LDS and tensor stores from LDS.
@@ -42,11 +50,20 @@ enum Stage : uint32_t {
RESERVED_10 = 10,
STAGE_LAST = RESERVED_10,
- ALL = 16,
- NUM_STAGE_SLOTS = ALL + 1
+ NUM_STAGES = STAGE_LAST + 1
};
-constexpr bool isValidStage(uint32_t S) { return S <= STAGE_LAST || S == ALL; }
+// Bits that a mask may legally set. Reserved stages are included: omitting a
+// stage whose operations do not exist yet is harmless, and accepting the bit
+// keeps masks portable as stages are filled in.
+constexpr uint32_t MaskAllStages = (uint32_t(1) << NUM_STAGES) - 1;
+
+constexpr bool isValidMask(uint32_t Mask) { return (Mask & ~MaskAllStages) == 0; }
+
+// A stage participates in an operation unless the mask names it.
+constexpr bool participates(uint32_t Mask, uint32_t S) {
+ return !(Mask & (uint32_t(1) << S));
+}
constexpr bool isReservedStage(uint32_t S) {
switch (S) {
@@ -62,7 +79,6 @@ constexpr bool isReservedStage(uint32_t S) {
case GLOBAL_LOAD_ASYNC_TO_LDS_MCAST:
case ASYNC_LDS_STORE:
case UNFORMATTED_BUFFER_GLOBAL_LOAD:
- case ALL:
return false;
}
llvm_unreachable("Unhandled stage");
@@ -92,12 +108,29 @@ constexpr const char *getStageName(uint32_t S) {
return "RESERVED_9";
case RESERVED_10:
return "RESERVED_10";
- case ALL:
- return "ALL";
}
llvm_unreachable("Unhandled stage");
}
+// Render the stages a mask leaves in as a '|'-separated list. Masks usually
+// name many more stages than they leave out, so listing the stages that
+// participate keeps the rendering short and says what actually happens.
+inline std::string getCoveredStagesString(uint32_t Mask) {
+ if (!(Mask & MaskAllStages))
+ return "all";
+ if ((Mask & MaskAllStages) == MaskAllStages)
+ return "none";
+ std::string Result;
+ for (uint32_t S = 0; S != NUM_STAGES; ++S) {
+ if (!participates(Mask, S))
+ continue;
+ if (!Result.empty())
+ Result += '|';
+ Result += getStageName(S);
+ }
+ return Result;
+}
+
} // namespace AsyncStage
} // namespace AMDGPU
} // namespace llvm
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 980c9fcada541..2e7250caebb3b 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -47,7 +47,6 @@
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
-#include "llvm/Support/AMDGPUAsyncStages.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
@@ -5134,14 +5133,14 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
return NewCall;
};
- // Legacy asyncmark intrinsics missed the stage operand. Append the ALL stage,
- // which is the behavior they had: every async operation belongs to it.
+ // Legacy asyncmark intrinsics missed the stage mask operand. Append an empty
+ // mask, which omits/ignores no stage and so preserves the behavior they had.
if ((F->getIntrinsicID() == Intrinsic::amdgcn_asyncmark &&
CI->arg_empty()) ||
(F->getIntrinsicID() == Intrinsic::amdgcn_wait_asyncmark &&
CI->arg_size() == 1)) {
SmallVector<Value *, 2> Args(CI->args());
- Args.push_back(Builder.getInt32(AMDGPU::AsyncStage::ALL));
+ Args.push_back(Builder.getInt32(0));
Function *NewDecl =
Intrinsic::getOrInsertDeclaration(F->getParent(), F->getIntrinsicID());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 07d72a3cd8591..3ee026c2162e9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -375,21 +375,23 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
if (MI->getOpcode() == AMDGPU::ASYNCMARK) {
if (isVerbose()) {
- OutStreamer->emitRawComment(" asyncmark(" +
- Twine(AMDGPU::AsyncStage::getStageName(
- MI->getOperand(0).getImm())) +
- ")");
+ OutStreamer->emitRawComment(
+ " asyncmark(stages=" +
+ Twine(AMDGPU::AsyncStage::getCoveredStagesString(
+ MI->getOperand(0).getImm())) +
+ ")");
}
return;
}
if (MI->getOpcode() == AMDGPU::WAIT_ASYNCMARK) {
if (isVerbose()) {
- OutStreamer->emitRawComment(" wait_asyncmark(" +
- Twine(MI->getOperand(0).getImm()) + ", " +
- Twine(AMDGPU::AsyncStage::getStageName(
- MI->getOperand(1).getImm())) +
- ")");
+ OutStreamer->emitRawComment(
+ " wait_asyncmark(" + Twine(MI->getOperand(0).getImm()) +
+ ", stages=" +
+ Twine(AMDGPU::AsyncStage::getCoveredStagesString(
+ MI->getOperand(1).getImm())) +
+ ")");
}
return;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 8c8b4079a8919..1ecc1a950e81f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -597,15 +597,14 @@ class WaitcntBrackets {
MCPhysReg Reg) const;
void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
AMDGPU::Waitcnt &Wait) const;
- AMDGPU::Waitcnt determineAsyncWait(unsigned N,
- AMDGPU::AsyncStage::Stage Stage);
+ AMDGPU::Waitcnt determineAsyncWait(unsigned N, uint32_t IgnoreMask);
void tryClearSCCWriteEvent(MachineInstr *Inst);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
void updateByEvent(HWEvents E, MachineInstr &MI);
- void recordAsyncMark(MachineInstr &MI, AMDGPU::AsyncStage::Stage Stage);
+ void recordAsyncMark(MachineInstr &MI, uint32_t OmitMask);
HWEvents getPendingEvents() const { return PendingEvents; }
bool hasPendingEvent() const { return PendingEvents.any(); }
@@ -708,8 +707,7 @@ class WaitcntBrackets {
static bool mergeScore(const MergeInfo &M, unsigned &Score,
unsigned OtherScore);
- bool mergeAsyncMarks(AMDGPU::AsyncStage::Stage Stage,
- ArrayRef<MergeInfo> MergeInfos,
+ bool mergeAsyncMarks(unsigned Stage, ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks);
iterator_range<MCRegUnitIterator> regunits(MCPhysReg Reg) const {
@@ -831,8 +829,7 @@ class WaitcntBrackets {
SmallVector<const MachineInstr *> LDSDMAStores;
// State of all counters at each async mark encountered so far, per stage.
- std::array<SmallVector<CounterValueArray, 0>,
- AMDGPU::AsyncStage::NUM_STAGE_SLOTS>
+ std::array<SmallVector<CounterValueArray, 0>, AMDGPU::AsyncStage::NUM_STAGES>
AsyncMarks;
// But in the rare pathological case, a nest of loops that pushes marks
@@ -843,7 +840,7 @@ class WaitcntBrackets {
// Track the upper bound score for async operations that are not part of a
// mark yet, per stage. Initialized to all zeros.
- std::array<CounterValueArray, AMDGPU::AsyncStage::NUM_STAGE_SLOTS> AsyncScore{};
+ std::array<CounterValueArray, AMDGPU::AsyncStage::NUM_STAGES> AsyncScore{};
};
SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
@@ -1112,10 +1109,8 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
setVMemScore(LDSDMA_BEGIN + Slot, T, CurrScore);
}
- if (auto Stage = Context->getAsyncStageToUpdate(Inst, T)) {
- AsyncScore[AMDGPU::AsyncStage::ALL][T] = CurrScore;
+ if (auto Stage = Context->getAsyncStageToUpdate(Inst, T))
AsyncScore[*Stage][T] = CurrScore;
- }
if (SIInstrInfo::isSBarrierSCCWrite(Inst.getOpcode())) {
setRegScore(AMDGPU::SCC, T, CurrScore);
@@ -1124,23 +1119,32 @@ void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
}
}
-void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
- AMDGPU::AsyncStage::Stage Stage) {
+void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst, uint32_t OmitMask) {
// In the absence of loops, AsyncMarks can grow linearly with the program
// until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
// limit every time we push a new mark, but that seems like unnecessary work
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
- AsyncMarks[Stage].push_back(AsyncScore[Stage]);
- LLVM_DEBUG({
- dbgs() << "recordAsyncMark(" << AMDGPU::AsyncStage::getStageName(Stage)
- << "):\n"
- << Inst;
- for (const auto &Mark : AsyncMarks[Stage]) {
- llvm::interleaveComma(Mark, dbgs());
- dbgs() << '\n';
- }
- });
+ //
+ // The mark joins every stage it does not omit. Each stage gets its own copy
+ // of the score, so the copies are consumed independently by later waits.
+ LLVM_DEBUG(dbgs() << "recordAsyncMark(stages="
+ << AMDGPU::AsyncStage::getCoveredStagesString(OmitMask)
+ << "):\n"
+ << Inst);
+ for (unsigned S = 0; S != AMDGPU::AsyncStage::NUM_STAGES; ++S) {
+ if (!AMDGPU::AsyncStage::participates(OmitMask, S))
+ continue;
+ AsyncMarks[S].push_back(AsyncScore[S]);
+ LLVM_DEBUG({
+ dbgs() << " stage " << AMDGPU::AsyncStage::getStageName(S) << ":\n";
+ for (const auto &Mark : AsyncMarks[S]) {
+ dbgs() << " ";
+ llvm::interleaveComma(Mark, dbgs());
+ dbgs() << '\n';
+ }
+ });
+ }
}
void WaitcntBrackets::print(raw_ostream &OS) const {
@@ -1243,9 +1247,7 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
}
OS << '\n';
- for (unsigned S = 0; S < AMDGPU::AsyncStage::NUM_STAGE_SLOTS; ++S) {
- if (!AMDGPU::AsyncStage::isValidStage(S))
- continue;
+ for (unsigned S = 0; S < AMDGPU::AsyncStage::NUM_STAGES; ++S) {
OS << "Async score (stage " << AMDGPU::AsyncStage::getStageName(S) << "): ";
if (llvm::all_of(AsyncScore[S], [](unsigned V) { return V == 0; }))
OS << "none";
@@ -1417,50 +1419,59 @@ void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
}
}
-AMDGPU::Waitcnt
-WaitcntBrackets::determineAsyncWait(unsigned N,
- AMDGPU::AsyncStage::Stage Stage) {
- auto &StageMarks = AsyncMarks[Stage];
- LLVM_DEBUG({
- dbgs() << "Need " << N << " async marks in stage "
- << AMDGPU::AsyncStage::getStageName(Stage) << ". Found "
- << StageMarks.size() << ":\n";
- for (const auto &Mark : StageMarks) {
- llvm::interleaveComma(Mark, dbgs());
- dbgs() << '\n';
+AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N,
+ uint32_t IgnoreMask) {
+ // Trim each stage the wait does not ignore down to at most N marks. The waits
+ // implied by the marks dropped from each stage accumulate into one Waitcnt,
+ // so a wait covering several stages is the union of their requirements.
+ AMDGPU::Waitcnt Wait;
+ for (unsigned S = 0; S != AMDGPU::AsyncStage::NUM_STAGES; ++S) {
+ if (!AMDGPU::AsyncStage::participates(IgnoreMask, S))
+ continue;
+
+ auto &StageMarks = AsyncMarks[S];
+ LLVM_DEBUG({
+ dbgs() << "Need " << N << " async marks in stage "
+ << AMDGPU::AsyncStage::getStageName(S) << ". Found "
+ << StageMarks.size() << ":\n";
+ for (const auto &Mark : StageMarks) {
+ llvm::interleaveComma(Mark, dbgs());
+ dbgs() << '\n';
+ }
+ });
+
+ unsigned StageN = N;
+ if (StageMarks.size() == MaxAsyncMarks) {
+ // Enforcing MaxAsyncMarks here is unnecessary work because the size of
+ // MaxAsyncMarks is linear when traversing straightline code. But we do
+ // need to check if truncation may have occured at a merge, and adjust N
+ // to ensure that a wait is generated.
+ LLVM_DEBUG(dbgs()
+ << "Possible truncation. Ensuring a non-trivial wait.\n");
+ StageN = std::min(StageN, (unsigned)MaxAsyncMarks - 1);
}
- });
- if (StageMarks.size() == MaxAsyncMarks) {
- // Enforcing MaxAsyncMarks here is unnecessary work because the size of
- // MaxAsyncMarks is linear when traversing straightline code. But we do
- // need to check if truncation may have occured at a merge, and adjust N
- // to ensure that a wait is generated.
- LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
- N = std::min(N, (unsigned)MaxAsyncMarks - 1);
- }
+ if (StageMarks.size() <= StageN) {
+ LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
+ continue;
+ }
- AMDGPU::Waitcnt Wait;
- if (StageMarks.size() <= N) {
- LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
- return Wait;
- }
+ size_t MarkIndex = StageMarks.size() - StageN - 1;
+ const auto &RequiredMark = StageMarks[MarkIndex];
+ for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
+ determineWaitForScore(T, RequiredMark[T], Wait);
+ }
- size_t MarkIndex = StageMarks.size() - N - 1;
- const auto &RequiredMark = StageMarks[MarkIndex];
- for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
- determineWaitForScore(T, RequiredMark[T], Wait);
+ // Immediately remove the waited mark and all older ones
+ // This happens BEFORE the wait is actually inserted, which is fine
+ // because we've already extracted the wait requirements
+ LLVM_DEBUG({
+ dbgs() << "Removing " << (MarkIndex + 1)
+ << " async marks after determining wait\n";
+ });
+ StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
}
- // Immediately remove the waited mark and all older ones
- // This happens BEFORE the wait is actually inserted, which is fine
- // because we've already extracted the wait requirements
- LLVM_DEBUG({
- dbgs() << "Removing " << (MarkIndex + 1)
- << " async marks after determining wait\n";
- });
- StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
-
LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
return Wait;
}
@@ -1747,7 +1758,7 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
unsigned N = II.getOperand(0).getImm();
LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, static_cast<AMDGPU::AsyncStage::Stage>(II.getOperand(1).getImm()));
+ N, II.getOperand(1).getImm());
Wait = Wait.combined(OldWait);
} else {
assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
@@ -2039,7 +2050,7 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
// shows up in the assembly as a comment with the original parameter N.
unsigned N = II.getOperand(0).getImm();
AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
- N, static_cast<AMDGPU::AsyncStage::Stage>(II.getOperand(1).getImm()));
+ N, II.getOperand(1).getImm());
Wait = Wait.combined(OldWait);
} else {
std::optional<AMDGPU::InstCounterType> CT =
@@ -2805,7 +2816,7 @@ bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
return OtherShifted > MyShifted;
}
-bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStage::Stage Stage,
+bool WaitcntBrackets::mergeAsyncMarks(unsigned Stage,
ArrayRef<MergeInfo> MergeInfos,
ArrayRef<CounterValueArray> OtherMarks) {
bool StrictDom = false;
@@ -2967,11 +2978,8 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
}
}
- for (unsigned S = 0; S != AMDGPU::AsyncStage::NUM_STAGE_SLOTS; ++S) {
- if (!AMDGPU::AsyncStage::isValidStage(S))
- continue;
- auto Stage = static_cast<AMDGPU::AsyncStage::Stage>(S);
- StrictDom |= mergeAsyncMarks(Stage, MergeInfos, Other.AsyncMarks[S]);
+ for (unsigned S = 0; S != AMDGPU::AsyncStage::NUM_STAGES; ++S) {
+ StrictDom |= mergeAsyncMarks(S, MergeInfos, Other.AsyncMarks[S]);
for (auto T : inst_counter_types(Context->MaxCounter))
StrictDom |=
mergeScore(MergeInfos[T], AsyncScore[S][T], Other.AsyncScore[S][T]);
@@ -3153,8 +3161,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// Asyncmarks record the current wait state and so should not allow
// waitcnts that occur after them to be merged into waitcnts that occur
// before.
- ScoreBrackets.recordAsyncMark(Inst,
- static_cast<AMDGPU::AsyncStage::Stage>(Inst.getOperand(0).getImm()));
+ ScoreBrackets.recordAsyncMark(Inst, Inst.getOperand(0).getImm());
continue;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index a4a2da25ed34a..dbe48dcfc9576 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5757,10 +5757,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
- int64_t Stage = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
- if (Stage < 0 || !AMDGPU::AsyncStage::isValidStage(Stage) ||
- AMDGPU::AsyncStage::isReservedStage(Stage)) {
- ErrInfo = "invalid asyncmark stage";
+ int64_t Mask = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
+ if (Mask < 0 || !AMDGPU::AsyncStage::isValidMask(Mask)) {
+ ErrInfo = "invalid asyncmark stage mask";
return false;
}
}
diff --git a/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
index 4997ad6d4c06d..5cf7adee9617b 100644
--- a/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
+++ b/llvm/test/Bitcode/upgrade-amdgcn-asyncmark.ll
@@ -2,15 +2,15 @@
; RUN: llvm-as < %t/legacy.ll | llvm-dis | FileCheck %s --check-prefix=LEGACY
; RUN: llvm-as < %t/staged.ll | llvm-dis | FileCheck %s --check-prefix=STAGED
-; The asyncmark intrinsics originally had no stage operand. Upgrade them to the
-; ALL stage (16), which is the behavior they had.
+; The asyncmark intrinsics originally had no stage mask operand. Upgrade them to
+; an empty mask, which leaves out no stage and so is the behavior they had.
;--- legacy.ll
define void @legacy() {
; LEGACY-LABEL: define void @legacy(
-; LEGACY-NEXT: call void @llvm.amdgcn.asyncmark(i32 16)
-; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
-; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
+; LEGACY-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+; LEGACY-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
; LEGACY-NEXT: ret void
;
call void @llvm.amdgcn.asyncmark()
@@ -20,14 +20,15 @@ define void @legacy() {
}
;--- staged.ll
-; Calls that already carry a stage operand are left alone.
+; Calls that already carry a mask are left alone. The masks here are non-empty
+; so that leaving them alone is distinguishable from upgrading them.
define void @staged() {
; STAGED-LABEL: define void @staged(
-; STAGED-NEXT: call void @llvm.amdgcn.asyncmark(i32 0)
-; STAGED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+; STAGED-NEXT: call void @llvm.amdgcn.asyncmark(i32 2046)
+; STAGED-NEXT: call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2046)
; STAGED-NEXT: ret void
;
- call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2046)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2046)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
index 2d0248dd9ae49..1a619048f5fb3 100644
--- a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
@@ -9,23 +9,23 @@ define float @raw.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %lds
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1, ALL)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -37,23 +37,23 @@ define float @raw.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace(3)
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1, ALL)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -65,23 +65,23 @@ define float @struct.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1, ALL)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -93,23 +93,23 @@ define float @struct.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace
; CHECK-NEXT: s_mov_b32 m0, s20
; CHECK-NEXT: v_mov_b32_e32 v0, 8
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:4 glc lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: buffer_load_dword v0, s[16:19], 0 idxen offset:8 slc lds
; CHECK-NEXT: v_mov_b32_e32 v0, s20
-; CHECK-NEXT: ; wait_asyncmark(1, ALL)
+; CHECK-NEXT: ; wait_asyncmark(1, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
@@ -124,13 +124,13 @@ define float @struct.ptr.buffer.load.vgpr.lds(ptr addrspace(8) inreg %rsrc, ptr
; CHECK-NEXT: s_mov_b32 m0, s4
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v1, s[16:19], 0 idxen lds
-; CHECK-NEXT: ; asyncmark(ALL)
+; CHECK-NEXT: ; asyncmark(stages=all)
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
main_body:
call void @llvm.amdgcn.struct.ptr.buffer.load.async.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 8, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
%res = load float, ptr addrspace(3) %lds
ret float %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 9c4cf0e5825b4..aaaffed23974d 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -21,18 +21,18 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
; SDAG-NEXT: ; wave barrier
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
; SDAG-NEXT: s_wait_loadcnt 0x2
@@ -60,7 +60,7 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
; GISEL-NEXT: ; wave barrier
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
@@ -69,11 +69,11 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
; GISEL-NEXT: s_wait_loadcnt 0x2
@@ -96,7 +96,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
call void @llvm.amdgcn.wave.barrier()
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load
%foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
@@ -108,15 +108,15 @@ entry:
call void @llvm.amdgcn.wave.barrier()
%bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
%bar_v12 = load i32, ptr addrspace(1) %bar_gep12
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds_gep21
; Wait for the next async mark.
; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds_gep22
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -151,11 +151,11 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b32 s7, s2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: .LBB1_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
@@ -163,8 +163,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: v_mov_b32_e32 v2, s8
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v2, s7
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(2, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s7, s7, 4
; SDAG-NEXT: s_cmp_lt_i32 s6, s3
@@ -175,14 +175,14 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_add_co_i32 s0, s0, -8
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; SDAG-NEXT: ds_load_b32 v0, v0
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
@@ -208,11 +208,11 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: s_mov_b32 s8, s2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: .LBB1_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
@@ -220,8 +220,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: v_mov_b32_e32 v1, s9
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v1, s8
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(2, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v1, v1
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -234,7 +234,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_lshl_b32 s0, s3, 2
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_add_co_u32 s0, s2, s0
; GISEL-NEXT: v_mov_b32_e32 v1, 0
@@ -243,7 +243,7 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GISEL-NEXT: ds_load_b32 v0, v0
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s2, v0
@@ -256,13 +256,13 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -274,10 +274,10 @@ loop_body:
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -290,14 +290,14 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
%sum_e2 = add i32 %sum_i, %lds_val_n_2
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -330,7 +330,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
@@ -338,7 +338,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_mov_b32 s8, 2
; SDAG-NEXT: s_mov_b32 s9, s10
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s12
; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
@@ -357,8 +357,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(2, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
@@ -376,7 +376,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_add_co_i32 s0, s11, -2
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -387,7 +387,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v1, v2, v1
; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
; SDAG-NEXT: s_wait_loadcnt 0x0
@@ -421,7 +421,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_mov_b32 s14, s10
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; GISEL-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
@@ -431,7 +431,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
; GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
; GISEL-NEXT: s_add_co_ci_u32 s7, s9, 0
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s15, v1
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -452,8 +452,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_mov_b32 s9, s19
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[6:7] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v3, s14
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(2, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: s_mov_b32 s12, s15
; GISEL-NEXT: ds_load_b32 v3, v3
@@ -480,7 +480,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
; GISEL-NEXT: s_add_co_i32 s4, s18, s9
@@ -496,7 +496,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: v_mov_b32_e32 v2, s1
; GISEL-NEXT: s_add_co_i32 s1, s8, s19
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0 offset:4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -512,7 +512,7 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
@@ -522,7 +522,7 @@ prolog:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -540,10 +540,10 @@ loop_body:
%cur_g = load i32, ptr addrspace(1) %bar_gep_cur
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
@@ -559,7 +559,7 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
@@ -568,7 +568,7 @@ epilog:
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
store i32 %sum_e2, ptr addrspace(1) %out_gep_e1
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
@@ -589,9 +589,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -604,9 +604,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
@@ -615,9 +615,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -636,11 +636,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_add_nc_u32_e32 v3, 4, v2
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -655,11 +655,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v2
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -668,11 +668,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
index 7ed261f9f4d34..cec31858c7703 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck %s
@@ -10,26 +11,26 @@
; Loop body: 18 markers
; CHECK-LABEL: test_loop_exceeds_max_first_iteration:
-; CHECK: ; wait_asyncmark(3, ALL)
+; CHECK: ; wait_asyncmark(3, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_exceeds_max_first_iteration(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
@@ -42,49 +43,49 @@ loop_header:
loop_body:
; Loop body with 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -95,26 +96,26 @@ exit:
; Loop body: 5 markers
; CHECK-LABEL: test_loop_needs_more_iterations:
-; CHECK: ; wait_asyncmark(3, ALL)
+; CHECK: ; wait_asyncmark(3, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(3)
define void @test_loop_needs_more_iterations(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
entry:
; Preloop: 5 async LDS DMA operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
@@ -127,20 +128,20 @@ loop_header:
loop_body:
; Loop body with 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_header
exit:
- call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 3, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -149,7 +150,7 @@ exit:
; Merge exceeds MaxAsyncMarkers
; CHECK-LABEL: max_when_merged:
-; CHECK: ; wait_asyncmark(17, ALL)
+; CHECK: ; wait_asyncmark(17, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(15)
define void @max_when_merged(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
@@ -160,68 +161,68 @@ entry:
then:
; 5 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %endif
else:
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %endif
endif:
- call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
@@ -230,53 +231,55 @@ endif:
; Straightline exceeds MaxAsyncMarkers
; CHECK-LABEL: no_max_in_straightline:
-; CHECK: ; wait_asyncmark(17, ALL)
+; CHECK: ; wait_asyncmark(17, stages=all)
; CHECK-NEXT: s_waitcnt vmcnt(17)
define void @no_max_in_straightline(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 %n, ptr addrspace(1) %out) {
; 18 async operations
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %in, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 17, i32 0)
%lds_val = load i32, ptr addrspace(3) %lds
store i32 %lds_val, ptr addrspace(1) %out
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 110935538509f..6e5db57591d24 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -13,7 +13,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: ds_read_b32 v1, v2 offset:4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v0, v1
@@ -25,7 +25,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: ds_read_b32 v1, v2 offset:4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v0, v1
@@ -34,7 +34,7 @@ define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addr
; GISEL-NEXT: s_setpc_b64 s[30:31]
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%val1 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val2 = load i32, ptr addrspace(3) %lds_gep1
%sum = add i32 %val1, %val2
store i32 %sum, ptr addrspace(3) %out
@@ -58,18 +58,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[3:4], off lds
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[3:4], off
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v4, v8, v7
@@ -91,18 +91,18 @@ define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr addrspace(3)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[3:4], off lds
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[3:4], off
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v4, v8, v7
@@ -119,7 +119,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -127,16 +127,16 @@ entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma
; This results in vmcnt(1), corresponding to %bar_v12. Could have been combined with the lgkmcnt(1) for %lds_val21.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -160,7 +160,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: v_mov_b32_e32 v8, 0x54
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -168,11 +168,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -193,7 +193,7 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: v_mov_b32_e32 v8, 0x54
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -201,11 +201,11 @@ define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr addrspa
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -222,7 +222,7 @@ entry:
%foo_v1 = load i32, ptr addrspace(1) %foo
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -230,16 +230,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -264,7 +264,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v0, v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v1, 0x58
; SDAG-NEXT: ; wave barrier
@@ -272,10 +272,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_dword v1, v[2:3], off
; SDAG-NEXT: v_mov_b32_e32 v2, s20
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: ds_read_b32 v3, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v6, v7, v6
@@ -297,7 +297,7 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v0, v[0:1], off
; GISEL-NEXT: v_mov_b32_e32 v1, 0x58
; GISEL-NEXT: ; wave barrier
@@ -305,10 +305,10 @@ define void @fence_with_asyncmark(ptr addrspace(8) inreg %buf, ptr addrspace(1)
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_dword v1, v[2:3], off
; GISEL-NEXT: v_mov_b32_e32 v2, s20
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: ds_read_b32 v3, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v6, v7, v6
@@ -326,7 +326,7 @@ entry:
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
fence release
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Second batch: global load, async global-to-LDS, global load.
%foo_v2 = load i32, ptr addrspace(1) %foo
@@ -334,16 +334,16 @@ entry:
call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf, ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
call void @llvm.amdgcn.wave.barrier()
%bar_v12 = load i32, ptr addrspace(1) %bar
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for first async mark and read from LDS.
; This results in vmcnt(3) corresponding to the second batch.
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val21 = load i32, ptr addrspace(3) %lds
; Wait for the next lds dma.
; This results in vmcnt(1) because the last global load is not async.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val22 = load i32, ptr addrspace(3) %lds
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
@@ -368,20 +368,20 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: .LBB4_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s7, v2
; SDAG-NEXT: s_mov_b32 m0, s7
; SDAG-NEXT: s_add_i32 s6, s6, 1
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(2, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: ds_read_b32 v6, v2
; SDAG-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -392,10 +392,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_cbranch_execnz .LBB4_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v0, v5, v0
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -409,21 +409,21 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 s7, 0
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v5, s7
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: .LBB4_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: v_readfirstlane_b32 s7, v2
; GISEL-NEXT: s_mov_b32 m0, s7
; GISEL-NEXT: s_add_i32 s6, s6, 1
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(2, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: ds_read_b32 v6, v2
; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
@@ -434,10 +434,10 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v0, v5, v0
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -446,11 +446,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -460,10 +460,10 @@ loop_body:
; Load next iteration
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -475,11 +475,11 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e2 = add i32 %sum_i, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e1 = add i32 %sum_e2, %lds_val_n_1
store i32 %sum_e2, ptr addrspace(1) %bar
@@ -500,12 +500,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: global_load_dword v14, v[3:4], off
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: global_load_dword v8, v[0:1], off
; SDAG-NEXT: global_load_dword v9, v[3:4], off
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_mov_b32_e32 v13, v8
; SDAG-NEXT: s_waitcnt vmcnt(1)
@@ -527,17 +527,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: v_mov_b32_e32 v10, v8
; SDAG-NEXT: v_mov_b32_e32 v14, v9
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(2, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(2, stages=all)
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_cbranch_execnz .LBB5_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: ds_read_b32 v0, v2
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(3)
; SDAG-NEXT: ds_read_b32 v1, v2
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v2, v2
; SDAG-NEXT: v_add_u32_e32 v3, v17, v16
@@ -561,12 +561,12 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: global_load_dword v14, v[3:4], off
; GISEL-NEXT: s_mov_b32 s6, 2
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: global_load_dword v8, v[0:1], off
; GISEL-NEXT: global_load_dword v9, v[3:4], off
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: v_mov_b32_e32 v13, v8
; GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -588,17 +588,17 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: v_mov_b32_e32 v10, v8
; GISEL-NEXT: v_mov_b32_e32 v14, v9
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(2, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(2, stages=all)
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GISEL-NEXT: ds_read_b32 v0, v2
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(3)
; GISEL-NEXT: ds_read_b32 v1, v2
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v3, v17, v16
@@ -617,13 +617,13 @@ prolog:
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Load second iteration
%v1 = load i32, ptr addrspace(1) %foo
%g1 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %loop_body
@@ -646,10 +646,10 @@ loop_body:
%cur_v = load i32, ptr addrspace(1) %foo
%cur_g = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
; Wait for iteration i-2 and process
- call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 2, i32 0)
%lds_idx = sub i32 %i, 2
%lds_val = load i32, ptr addrspace(3) %lds
@@ -662,13 +662,13 @@ loop_body:
epilog:
; Process remaining iterations
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_val_n_2 = load i32, ptr addrspace(3) %lds
%sum_e0 = add i32 %sum, %g1_phi
%sum_e1 = add i32 %v1_phi, %sum_e0
%sum_e2 = add i32 %sum_e1, %lds_val_n_2
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_val_n_1 = load i32, ptr addrspace(3) %lds
%sum_e3 = add i32 %cur_v, %cur_g
%sum_e4 = add i32 %sum_e3, %lds_val_n_1
@@ -689,9 +689,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -706,9 +706,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -717,9 +717,9 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
@@ -739,12 +739,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: v_readfirstlane_b32 s4, v5
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: s_mov_b32 m0, s4
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; asyncmark(ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; asyncmark(stages=all)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -761,12 +761,12 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: v_readfirstlane_b32 s4, v5
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; asyncmark(ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; asyncmark(stages=all)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -776,11 +776,11 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%val = load i32, ptr addrspace(3) %lds
store i32 %val, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
index 0111a251d4415..f49c86e0cc420 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.ll
@@ -1,52 +1,30 @@
; RUN: split-file %s %t
-; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/oob.ll 2>&1 | FileCheck %s -check-prefix=OOB
-; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/oob-wait.ll 2>&1 | FileCheck %s -check-prefix=OOB-WAIT
-; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/reserved.ll 2>&1 | FileCheck %s -check-prefix=RESERVED
-; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/reserved-wait.ll 2>&1 | FileCheck %s -check-prefix=RESERVED-WAIT
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/mark.ll 2>&1 | FileCheck %s -check-prefix=MARK
+; RUN: not llc -mtriple=amdgpu12.50 -filetype=null %t/wait.ll 2>&1 | FileCheck %s -check-prefix=WAIT
-; The stage operand carries a RangeSet covering the valid, non-reserved stages,
-; so the IR verifier rejects bad stages before instruction selection. Reserved
-; stages are simply the gaps in that set.
+; The stage mask operand carries a Range covering the bits of the defined
+; stages, so the IR verifier rejects a mask naming a stage that does not exist
+; before instruction selection. Bits of reserved stages are inside the range and
+; so are accepted: omitting a stage whose operations do not exist yet is
+; harmless, and keeps masks portable as the reserved slots are filled in.
-;--- oob.ll
-; Values above ALL, and values between STAGE_LAST and ALL, are out of range.
-; OOB: immarg value 11 for arg 0 out of range set
-; OOB: immarg value 17 for arg 0 out of range set
-define amdgpu_kernel void @asyncmark_out_of_range() {
- call void @llvm.amdgcn.asyncmark(i32 11)
- call void @llvm.amdgcn.asyncmark(i32 17)
+;--- mark.ll
+; 2048 is the bit just past the last stage, and 4096 is further out still.
+; MARK: immarg value 2048 for arg 0 out of range [0,2048)
+; MARK: immarg value 4096 for arg 0 out of range [0,2048)
+define amdgpu_kernel void @asyncmark_mask_out_of_range() {
+ call void @llvm.amdgcn.asyncmark(i32 2048)
+ call void @llvm.amdgcn.asyncmark(i32 4096)
ret void
}
-;--- oob-wait.ll
-; OOB-WAIT: immarg value 15 for arg 1 out of range set
-define amdgpu_kernel void @wait_asyncmark_out_of_range() {
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 15)
- ret void
-}
-
-;--- reserved.ll
-; Every reserved stage is rejected. These hold slots in the stage numbering for
-; async operations that do not exist yet.
-; RESERVED: immarg value 4 for arg 0 out of range set
-; RESERVED: immarg value 6 for arg 0 out of range set
-; RESERVED: immarg value 7 for arg 0 out of range set
-; RESERVED: immarg value 8 for arg 0 out of range set
-; RESERVED: immarg value 9 for arg 0 out of range set
-; RESERVED: immarg value 10 for arg 0 out of range set
-define amdgpu_kernel void @asyncmark_reserved() {
- call void @llvm.amdgcn.asyncmark(i32 4)
- call void @llvm.amdgcn.asyncmark(i32 6)
- call void @llvm.amdgcn.asyncmark(i32 7)
- call void @llvm.amdgcn.asyncmark(i32 8)
- call void @llvm.amdgcn.asyncmark(i32 9)
- call void @llvm.amdgcn.asyncmark(i32 10)
- ret void
-}
-
-;--- reserved-wait.ll
-; RESERVED-WAIT: immarg value 4 for arg 1 out of range set
-define amdgpu_kernel void @wait_asyncmark_reserved() {
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 4)
+;--- wait.ll
+; A mask made only of reserved stage bits is fine, so the sole error here is the
+; out-of-range one.
+; WAIT: immarg value 2048 for arg 1 out of range [0,2048)
+; WAIT-NOT: out of range
+define amdgpu_kernel void @wait_asyncmark_mask_out_of_range() {
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2000)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2048)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
deleted file mode 100644
index 05d8c7e9c00a1..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-err.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# RUN: split-file %s %t
-# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none -verify-machineinstrs %t/mark.mir -o /dev/null 2>&1 | FileCheck %s -check-prefix=MARK
-# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none -verify-machineinstrs %t/wait.mir -o /dev/null 2>&1 | FileCheck %s -check-prefix=WAIT
-
-#--- mark.mir
-# A reserved stage is as invalid as an out-of-range one.
-# MARK: *** Bad machine code: invalid asyncmark stage ***
-# MARK-NEXT: - function: asyncmark_reserved_stage
-# MARK: ASYNCMARK 4
----
-name: asyncmark_reserved_stage
-tracksRegLiveness: true
-body: |
- bb.0:
- ASYNCMARK 4
- S_ENDPGM 0
-...
-
-#--- wait.mir
-# WAIT: *** Bad machine code: invalid asyncmark stage ***
-# WAIT-NEXT: - function: wait_asyncmark_out_of_range
-# WAIT: WAIT_ASYNCMARK 0, 17
----
-name: wait_asyncmark_out_of_range
-tracksRegLiveness: true
-body: |
- bb.0:
- WAIT_ASYNCMARK 0, 17
- S_ENDPGM 0
-...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
index 6a054369559b3..10c861e469f17 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-max.mir
@@ -5,9 +5,10 @@
# mark sequence. Saturating one stage must not perturb another.
---
-# Stage 1 receives 20 marks, well past the cap, while stage 0 receives exactly
-# one. The wait on stage 0 must still find its single mark and produce an exact
-# tensorcnt wait: it is not affected by stage 1's truncation.
+# Mask 2045 covers only GLOBAL_LOAD_ASYNC_TO_LDS, which receives 20 marks, well
+# past the cap. Mask 2046 covers only TENSOR, which receives exactly one. The
+# wait covering TENSOR must still find its single mark and produce an exact
+# tensorcnt wait: it is not affected by the other stage's truncation.
name: saturate_one_stage_only
tracksRegLiveness: true
machineFunctionInfo:
@@ -22,95 +23,95 @@ body: |
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: ASYNCMARK 2046
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 0
+ ; CHECK-NEXT: ASYNCMARK 2045
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2046
; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
; CHECK-NEXT: S_ENDPGM 0
TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ASYNCMARK 0
+ ASYNCMARK 2046
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
- ; Stage 0 has one mark; counting back 0 reaches it exactly.
- WAIT_ASYNCMARK 0, 0
+ ; TENSOR has one mark; counting back 0 reaches it exactly.
+ WAIT_ASYNCMARK 0, 2046
S_ENDPGM 0
...
---
@@ -130,89 +131,89 @@ body: |
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
- ; CHECK-NEXT: WAIT_ASYNCMARK 18, 1
+ ; CHECK-NEXT: ASYNCMARK 2045
+ ; CHECK-NEXT: WAIT_ASYNCMARK 18, 2045
; CHECK-NEXT: S_WAIT_ASYNCCNT 18, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
- WAIT_ASYNCMARK 18, 1
+ WAIT_ASYNCMARK 18, 2045
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
index c5f20d94e14ba..b5452b00f9259 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-merge.mir
@@ -13,8 +13,8 @@
# a counterpart in the other predecessor, so both survive the join as
# single-element sequences padded with the zero-mark identity.
#
-# The join then waits on stage 1 only. The tensor DMA from bb.2 must stay in
-# flight: no S_WAIT_TENSORCNT may appear.
+# The join then waits with a mask covering GLOBAL_LOAD_ASYNC_TO_LDS only. The
+# tensor DMA from bb.2 must stay in flight: no S_WAIT_TENSORCNT may appear.
name: merge_disjoint_stages
tracksRegLiveness: true
machineFunctionInfo:
@@ -35,7 +35,7 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -43,11 +43,11 @@ body: |
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: ASYNCMARK 2046
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2045
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -56,24 +56,24 @@ body: |
S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
S_CBRANCH_SCC1 %bb.2, implicit killed $scc
- ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) on ASYNC_CNT.
+ ; Mask 2045 covers GLOBAL_LOAD_ASYNC_TO_LDS only, which is on ASYNC_CNT.
bb.1:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
S_BRANCH %bb.3
- ; Stage 0 (TENSOR) on TENSOR_CNT.
+ ; Mask 2046 covers TENSOR only, which is on TENSOR_CNT.
bb.2:
liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ASYNCMARK 0
+ ASYNCMARK 2046
S_BRANCH %bb.3
bb.3:
- WAIT_ASYNCMARK 0, 1
+ WAIT_ASYNCMARK 0, 2045
S_ENDPGM 0
...
---
@@ -100,9 +100,9 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: ASYNCMARK 2046
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -110,12 +110,12 @@ body: |
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ; CHECK-NEXT: ASYNCMARK 0
+ ; CHECK-NEXT: ASYNCMARK 2046
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 0
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2046
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2045
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_WAIT_TENSORCNT 0, implicit-def $tensorcnt, implicit $tensorcnt
; CHECK-NEXT: S_ENDPGM 0
@@ -125,27 +125,27 @@ body: |
S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
S_CBRANCH_SCC1 %bb.2, implicit killed $scc
- ; Marks in both stage 0 and stage 1.
+ ; Marks in both TENSOR and GLOBAL_LOAD_ASYNC_TO_LDS.
bb.1:
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ASYNCMARK 0
+ ASYNCMARK 2046
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
S_BRANCH %bb.3
- ; Marks in stage 0 only; stage 1's sequence is empty on this path.
+ ; Marks in TENSOR only; GLOBAL_LOAD_ASYNC_TO_LDS's sequence is empty here.
bb.2:
liveins: $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
TENSOR_LOAD_TO_LDS_d2 $sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, implicit-def $tensorcnt, implicit $exec, implicit $tensorcnt
- ASYNCMARK 0
+ ASYNCMARK 2046
S_BRANCH %bb.3
bb.3:
- WAIT_ASYNCMARK 0, 0
- WAIT_ASYNCMARK 0, 1
+ WAIT_ASYNCMARK 0, 2046
+ WAIT_ASYNCMARK 0, 2045
S_ENDPGM 0
...
---
@@ -172,9 +172,9 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -182,11 +182,11 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2045
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -195,38 +195,39 @@ body: |
S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
S_CBRANCH_SCC1 %bb.2, implicit killed $scc
- ; Two marks in stage 1.
+ ; Two marks in GLOBAL_LOAD_ASYNC_TO_LDS.
bb.1:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
S_BRANCH %bb.3
- ; One mark in stage 1.
+ ; One mark in GLOBAL_LOAD_ASYNC_TO_LDS.
bb.2:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
S_BRANCH %bb.3
bb.3:
- WAIT_ASYNCMARK 0, 1
+ WAIT_ASYNCMARK 0, 2045
S_ENDPGM 0
...
---
-# A stage-specific sequence and the ALL sequence merged at the same join. ALL is
-# held separately from the dense stage array, so it must take part in the merge
-# on the same footing.
-name: merge_stage_and_all
+# A wide mark and a narrow one merged at the same join. bb.1's empty mask joins
+# every stage, so it lands in GLOBAL_LOAD_ASYNC_TO_LDS's sequence alongside
+# bb.2's narrow mark, and those two merge against each other. bb.1's copies in
+# the other ten stages have no counterpart in bb.2 and are padded instead.
+name: merge_wide_and_narrow_marks
tracksRegLiveness: true
machineFunctionInfo:
occupancy: 8
body: |
- ; CHECK-LABEL: name: merge_stage_and_all
+ ; CHECK-LABEL: name: merge_wide_and_narrow_marks
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
@@ -241,7 +242,7 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 16
+ ; CHECK-NEXT: ASYNCMARK 0
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
@@ -249,12 +250,12 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK 1
+ ; CHECK-NEXT: ASYNCMARK 2045
; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 16
- ; CHECK-NEXT: WAIT_ASYNCMARK 0, 1
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 0
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0, 2045
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -267,18 +268,18 @@ body: |
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 16
+ ASYNCMARK 0
S_BRANCH %bb.3
bb.2:
liveins: $vgpr0_vgpr1, $vgpr2
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK 1
+ ASYNCMARK 2045
S_BRANCH %bb.3
bb.3:
- WAIT_ASYNCMARK 0, 16
- WAIT_ASYNCMARK 0, 1
+ WAIT_ASYNCMARK 0, 0
+ WAIT_ASYNCMARK 0, 2045
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
index 2a988b4426eab..fcc6b37e79ec2 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage-pregfx12.ll
@@ -5,8 +5,8 @@
; Stage UNFORMATTED_BUFFER_GLOBAL_LOAD maps to LOAD_CNT, which pre-gfx12 is
; vmcnt. Buffer async LDS loads are the operations in that stage.
-; Marks in stage 5 form their own sequence; wait_asyncmark(1, 5) counts back
-; through it and produces a vmcnt wait, not a blanket wait.
+; Marks covering that stage form their own sequence; a wait covering it counts
+; back through them and produces a vmcnt wait, not a blanket wait.
define amdgpu_kernel void @stage_unformatted_buffer_load(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
; SDAG-LABEL: stage_unformatted_buffer_load:
@@ -18,13 +18,13 @@ define amdgpu_kernel void @stage_unformatted_buffer_load(<4 x i32> %rsrc, ptr ad
; SDAG-NEXT: s_mov_b32 m0, s6
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
-; SDAG-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ; asyncmark(stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
-; SDAG-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ; asyncmark(stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
; SDAG-NEXT: v_mov_b32_e32 v0, s6
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
-; SDAG-NEXT: ; wait_asyncmark(1, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: ds_read_b32 v0, v0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -40,13 +40,13 @@ define amdgpu_kernel void @stage_unformatted_buffer_load(<4 x i32> %rsrc, ptr ad
; GISEL-NEXT: s_mov_b32 m0, s6
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
-; GISEL-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ; asyncmark(stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:4 glc lds
-; GISEL-NEXT: ; asyncmark(UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ; asyncmark(stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 offset:8 slc lds
; GISEL-NEXT: v_mov_b32_e32 v0, s6
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
-; GISEL-NEXT: ; wait_asyncmark(1, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: ds_read_b32 v0, v0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -54,17 +54,17 @@ define amdgpu_kernel void @stage_unformatted_buffer_load(<4 x i32> %rsrc, ptr ad
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 5)
+ call void @llvm.amdgcn.asyncmark(i32 2015)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 4, i32 1)
- call void @llvm.amdgcn.asyncmark(i32 5)
+ call void @llvm.amdgcn.asyncmark(i32 2015)
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 8, i32 2)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 5)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2015)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %out
ret void
}
-; A wait on a stage that never received a mark is a no-op, even though the
+; A wait covering a stage that never received a mark is a no-op, even though the
; outstanding buffer loads are on the counter that stage would map to.
define amdgpu_kernel void @wait_on_empty_stage(<4 x i32> %rsrc, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
@@ -78,8 +78,8 @@ define amdgpu_kernel void @wait_on_empty_stage(<4 x i32> %rsrc, ptr addrspace(3)
; SDAG-NEXT: v_mov_b32_e32 v0, s6
; SDAG-NEXT: buffer_load_dword off, s[0:3], 0 lds
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(0, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; SDAG-NEXT: ds_read_b32 v0, v0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v1, v0, s[0:1]
@@ -95,16 +95,16 @@ define amdgpu_kernel void @wait_on_empty_stage(<4 x i32> %rsrc, ptr addrspace(3)
; GISEL-NEXT: v_mov_b32_e32 v0, s6
; GISEL-NEXT: buffer_load_dword off, s[0:3], 0 lds
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3c
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(0, UNFORMATTED_BUFFER_GLOBAL_LOAD)
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=UNFORMATTED_BUFFER_GLOBAL_LOAD)
; GISEL-NEXT: ds_read_b32 v0, v0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.raw.buffer.load.async.lds(<4 x i32> %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 5)
+ call void @llvm.amdgcn.asyncmark(i32 2046)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2015)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
index bf4b2e3927463..6bbad988472b4 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-stage.ll
@@ -1,15 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=SDAG
-; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GISEL
+; RUN: llc -verify-machineinstrs -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -verify-machineinstrs -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GISEL
-; Asyncmark stages partition the mark sequence. A wait on one stage must not
-; observe counters belonging to another stage, and must not count back through
-; marks that belong to another stage even when the two stages share a counter.
+; Each async stage has its own sequence of marks. The intrinsics take a mask of
+; the stages to leave out: asyncmark omits them, wait_asyncmark disregards them.
+; The masks below are written as the complement of the stages they cover, e.g.
+; 2046 = 0x7fe omits every stage but TENSOR.
; The over-waiting fix. Tensor DMA (TENSOR_CNT) and async global-to-LDS
-; (ASYNC_CNT) are interleaved, each with its own marks. wait_asyncmark(0, TENSOR)
-; must wait only on tensorcnt: the outstanding async load is unrelated to it and
-; must stay in flight.
+; (ASYNC_CNT) are interleaved, each marked into its own stage. A wait covering
+; TENSOR alone must wait only on tensorcnt: the outstanding async load is
+; unrelated to it and must stay in flight.
define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
; SDAG-LABEL: wait_tensor_ignores_async:
@@ -25,10 +26,10 @@ define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %t
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
-; SDAG-NEXT: ; asyncmark(TENSOR)
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; SDAG-NEXT: ; wait_asyncmark(0, TENSOR)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
; SDAG-NEXT: s_wait_tensorcnt 0x0
; SDAG-NEXT: ds_load_b32 v1, v1
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -48,10 +49,10 @@ define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %t
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
-; GISEL-NEXT: ; asyncmark(TENSOR)
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; GISEL-NEXT: ; wait_asyncmark(0, TENSOR)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
; GISEL-NEXT: s_wait_tensorcnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -59,19 +60,19 @@ define amdgpu_kernel void @wait_tensor_ignores_async(<4 x i32> %sd, <8 x i32> %t
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2046)
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
; Waits for the tensor DMA only. No s_wait_asynccnt may be emitted here.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2046)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %glb
ret void
}
-; Same shape, but waiting on the async stage instead. Now asynccnt is required
+; Same shape, but covering the async stage instead. Now asynccnt is required
; and tensorcnt is not.
define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
@@ -88,10 +89,10 @@ define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %t
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
-; SDAG-NEXT: ; asyncmark(TENSOR)
-; SDAG-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v1, v1
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -111,10 +112,10 @@ define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %t
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
-; GISEL-NEXT: ; asyncmark(TENSOR)
-; GISEL-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v1, v1
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -122,19 +123,19 @@ define amdgpu_kernel void @wait_async_ignores_tensor(<4 x i32> %sd, <8 x i32> %t
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2046)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 1)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2045)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %glb
ret void
}
; Two stages that share ASYNC_CNT. Stages are still independent mark sequences:
-; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS) counts back through the loads only,
+; a wait covering GLOBAL_LOAD_ASYNC_TO_LDS counts back through the loads only,
; so the intervening store mark does not shift the wait. Both loads and the store
; are on asynccnt, so a wait is emitted, but its value reflects one sequence.
@@ -154,12 +155,12 @@ define amdgpu_kernel void @same_counter_distinct_sequences(ptr addrspace(1) %glb
; SDAG-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; SDAG-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(ASYNC_LDS_STORE)
+; SDAG-NEXT: ; asyncmark(stages=ASYNC_LDS_STORE)
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; SDAG-NEXT: ; wait_asyncmark(1, GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: ds_load_b32 v1, v1
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -181,12 +182,12 @@ define amdgpu_kernel void @same_counter_distinct_sequences(ptr addrspace(1) %glb
; GISEL-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; GISEL-NEXT: global_store_async_from_lds_b32 v0, v2, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(ASYNC_LDS_STORE)
+; GISEL-NEXT: ; asyncmark(stages=ASYNC_LDS_STORE)
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; GISEL-NEXT: ; wait_asyncmark(1, GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(1, stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v1, v1
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -196,31 +197,31 @@ entry:
%lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%lds2 = getelementptr i32, ptr addrspace(3) %lds, i32 2
- ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) mark #0.
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark #0.
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
- ; Stage 3 (ASYNC_LDS_STORE) mark #0. Same counter, different sequence.
+ ; ASYNC_LDS_STORE mark #0. Same counter, different sequence.
call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 3)
+ call void @llvm.amdgcn.asyncmark(i32 2039)
- ; Stage 1 mark #1.
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark #1.
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds2, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
- ; Counts back one mark in stage 1's sequence, skipping the stage 3 mark.
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 1)
+ ; Counts back one mark in the load stage, skipping the store stage's mark.
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 2045)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %glb
ret void
}
-; Stage ALL is the pre-stage behavior: a mark observes every counter and a wait
-; on it waits for all of them. Marks placed in ALL are their own sequence, so
-; stage-specific marks do not perturb it.
+; An empty mask omits nothing, so the mark joins every stage and the wait covers
+; every stage. This is the behavior of the original stage-less intrinsics, and is
+; what old IR auto-upgrades to.
-define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
-; SDAG-LABEL: stage_all_waits_for_everything:
+define amdgpu_kernel void @empty_mask_covers_everything(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: empty_mask_covers_everything:
; SDAG: ; %bb.0: ; %entry
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_mov_b64 s[64:65], 0
@@ -234,8 +235,8 @@ define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i3
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
-; SDAG-NEXT: ; asyncmark(ALL)
-; SDAG-NEXT: ; wait_asyncmark(0, ALL)
+; SDAG-NEXT: ; asyncmark(stages=all)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=all)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: s_wait_tensorcnt 0x0
; SDAG-NEXT: ds_load_b32 v1, v1
@@ -243,7 +244,7 @@ define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i3
; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
; SDAG-NEXT: s_endpgm
;
-; GISEL-LABEL: stage_all_waits_for_everything:
+; GISEL-LABEL: empty_mask_covers_everything:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b64 s[64:65], 0
@@ -257,8 +258,8 @@ define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i3
; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
-; GISEL-NEXT: ; asyncmark(ALL)
-; GISEL-NEXT: ; wait_asyncmark(0, ALL)
+; GISEL-NEXT: ; asyncmark(stages=all)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=all)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: s_wait_tensorcnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0
@@ -268,15 +269,16 @@ define amdgpu_kernel void @stage_all_waits_for_everything(<4 x i32> %sd, <8 x i3
entry:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %glb
ret void
}
-; A wait on a stage with no marks at all is a no-op: nothing to count back to.
+; A wait covering a stage with no marks at all is a no-op: nothing to count back
+; to.
define amdgpu_kernel void @wait_on_empty_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
; SDAG-LABEL: wait_on_empty_stage:
@@ -289,8 +291,8 @@ define amdgpu_kernel void @wait_on_empty_stage(ptr addrspace(1) %glb, ptr addrsp
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; SDAG-NEXT: ; wait_asyncmark(0, TENSOR)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
; SDAG-NEXT: ds_load_b32 v1, v1
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -306,25 +308,25 @@ define amdgpu_kernel void @wait_on_empty_stage(ptr addrspace(1) %glb, ptr addrsp
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
-; GISEL-NEXT: ; wait_asyncmark(0, TENSOR)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
; GISEL-NEXT: ds_load_b32 v0, v0
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
- ; Stage 0 (TENSOR) never had a mark, so this waits for nothing.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
+ ; TENSOR never had a mark, so this waits for nothing.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2046)
%v = load i32, ptr addrspace(3) %lds
store i32 %v, ptr addrspace(1) %glb
ret void
}
; The multicast (cluster) loads have their own stage, distinct from the plain
-; async loads to LDS even though both use the async counter. A wait on the
+; async loads to LDS even though both use the async counter. A wait covering the
; multicast stage must not count back through a plain load's mark.
define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 %mask) {
; SDAG-LABEL: mcast_is_a_distinct_stage:
@@ -341,10 +343,10 @@ define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr
; SDAG-NEXT: v_mov_b32_e32 v2, s2
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; SDAG-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
-; SDAG-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
-; SDAG-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v1, v1 offset:4
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -365,10 +367,10 @@ define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr
; GISEL-NEXT: v_mov_b32_e32 v2, s2
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
; GISEL-NEXT: cluster_load_async_to_lds_b32 v2, v0, s[0:1] offset:4
-; GISEL-NEXT: ; asyncmark(GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
-; GISEL-NEXT: ; wait_asyncmark(0, GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=GLOBAL_LOAD_ASYNC_TO_LDS_MCAST)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v1, v1 offset:4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -377,21 +379,169 @@ define amdgpu_kernel void @mcast_is_a_distinct_stage(ptr addrspace(1) %glb, ptr
entry:
%lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
- ; Stage 1 (GLOBAL_LOAD_ASYNC_TO_LDS) mark.
+ ; GLOBAL_LOAD_ASYNC_TO_LDS mark.
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 1)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
- ; Stage 2 (GLOBAL_LOAD_ASYNC_TO_LDS_MCAST) mark. Same counter, own sequence.
+ ; GLOBAL_LOAD_ASYNC_TO_LDS_MCAST mark. Same counter, own sequence.
call void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0, i32 %mask)
- call void @llvm.amdgcn.asyncmark(i32 2)
+ call void @llvm.amdgcn.asyncmark(i32 2043)
- ; Drains stage 2 only. The stage 1 mark is in a different sequence.
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2)
+ ; Drains the multicast stage only. The plain load's mark is in another
+ ; sequence.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2043)
%v = load i32, ptr addrspace(3) %lds1
store i32 %v, ptr addrspace(1) %glb
ret void
}
+; A mark that omits nothing relevant joins several stages at once, and each
+; stage keeps its own copy. Here one mark covers both TENSOR and
+; GLOBAL_LOAD_ASYNC_TO_LDS, so a wait on either one alone finds it.
+
+define amdgpu_kernel void @one_mark_joins_two_stages(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: one_mark_joins_two_stages:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=TENSOR|GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v1
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: one_mark_joins_two_stages:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=TENSOR|GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR)
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v0, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+
+ ; Joins TENSOR and GLOBAL_LOAD_ASYNC_TO_LDS.
+ call void @llvm.amdgcn.asyncmark(i32 2044)
+
+ ; Consumes the TENSOR copy. The GLOBAL_LOAD_ASYNC_TO_LDS copy is untouched, so
+ ; only tensorcnt is waited on.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2046)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
+; A wait covering several stages is the union of their requirements. The two
+; stages here sit on different counters, so both a tensorcnt and an asynccnt
+; wait are emitted, each derived from its own sequence.
+
+define amdgpu_kernel void @multi_stage_wait_unions_counters(<4 x i32> %sd, <8 x i32> %td, ptr addrspace(1) %glb, ptr addrspace(3) %lds) {
+; SDAG-LABEL: multi_stage_wait_unions_counters:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-NEXT: v_nop
+; SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-NEXT: s_clause 0x2
+; SDAG-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: s_add_co_i32 s4, s18, 4
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; SDAG-NEXT: ; asyncmark(stages=TENSOR)
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_store_async_from_lds_b32 v0, v1, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=ASYNC_LDS_STORE)
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[16:17] offset:4
+; SDAG-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; SDAG-NEXT: ; wait_asyncmark(0, stages=TENSOR|ASYNC_LDS_STORE)
+; SDAG-NEXT: s_wait_asynccnt 0x1
+; SDAG-NEXT: s_wait_tensorcnt 0x0
+; SDAG-NEXT: ds_load_b32 v1, v2
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v0, v1, s[16:17]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: multi_stage_wait_unions_counters:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GISEL-NEXT: v_nop
+; GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GISEL-NEXT: s_clause 0x2
+; GISEL-NEXT: s_load_b96 s[16:18], s[4:5], 0x64 nv
+; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x44 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: s_add_co_u32 s4, s18, 4
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GISEL-NEXT: v_mov_b32_e32 v2, s18
+; GISEL-NEXT: tensor_load_to_lds s[0:3], s[8:15]
+; GISEL-NEXT: ; asyncmark(stages=TENSOR)
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_store_async_from_lds_b32 v0, v1, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=ASYNC_LDS_STORE)
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v0, s[16:17] offset:4
+; GISEL-NEXT: ; asyncmark(stages=GLOBAL_LOAD_ASYNC_TO_LDS)
+; GISEL-NEXT: ; wait_asyncmark(0, stages=TENSOR|ASYNC_LDS_STORE)
+; GISEL-NEXT: s_wait_asynccnt 0x1
+; GISEL-NEXT: s_wait_tensorcnt 0x0
+; GISEL-NEXT: ds_load_b32 v1, v2
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v0, v1, s[16:17]
+; GISEL-NEXT: s_endpgm
+entry:
+ %lds1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+
+ call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %sd, <8 x i32> %td, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2046)
+
+ call void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2039)
+
+ ; This async load is marked into neither stage, so neither wait covers it.
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %glb, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark(i32 2045)
+
+ ; Covers TENSOR and ASYNC_LDS_STORE together.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 2038)
+ %v = load i32, ptr addrspace(3) %lds
+ store i32 %v, ptr addrspace(1) %glb
+ ret void
+}
+
declare void @llvm.amdgcn.asyncmark(i32)
declare void @llvm.amdgcn.wait.asyncmark(i16, i32)
declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32>, <8 x i32>, <4 x i32>, <4 x i32>, <8 x i32>, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
index 035544d34bd40..c0ba4e1f95782 100644
--- a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
+++ b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
@@ -35,7 +35,7 @@ body: |
# WAIT_ASYNCMARK is a meta instruction that emits zero bytes.
# codeLenInByte = s_waitcnt (4) + WAIT_ASYNCMARK (0) = 4.
# CHECK: wait_asyncmark_meta: ; @wait_asyncmark_meta
-# CHECK: ; wait_asyncmark(1, ALL)
+# CHECK: ; wait_asyncmark(1, stages=all)
# CHECK: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
# CHECK: ; codeLenInByte = 4
---
@@ -44,7 +44,7 @@ tracksRegLiveness: true
body: |
bb.0:
- WAIT_ASYNCMARK 1, 16
+ WAIT_ASYNCMARK 1, 0
...
# CHECK: align4: ; @align4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
index a65dacd1f9067..138f877ef59e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -331,13 +331,13 @@ define amdgpu_ps void @tensor_load_to_lds_with_asyncmark(<4 x i32> inreg %D0, <8
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark(ALL)
-; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -349,13 +349,13 @@ define amdgpu_ps void @tensor_store_from_lds_with_asyncmark(<4 x i32> inreg %D0,
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark(ALL)
-; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -370,14 +370,14 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
-; GFX1250-NEXT: ; asyncmark(ALL)
+; GFX1250-NEXT: ; asyncmark(stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0xa
; GFX1250-NEXT: tensor_load_to_lds s[12:15], s[16:23]
-; GFX1250-NEXT: ; asyncmark(ALL)
-; GFX1250-NEXT: ; wait_asyncmark(1, ALL)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x1
; GFX1250-NEXT: ds_load_b32 v1, v0
-; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: ds_load_b32 v2, v0 offset:4
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -385,15 +385,15 @@ define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <
; GFX1250-NEXT: ds_store_b32 v0, v1
; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0a, <8 x i32> %D1a, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0b, <8 x i32> %D1b, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
%lds_v0 = load i32, ptr addrspace(3) %lds
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%lds_v1 = load i32, ptr addrspace(3) %lds_gep1
@@ -415,15 +415,15 @@ define void @tensor_and_async_lds_with_asyncmark(<4 x i32> inreg %D0, <8 x i32>
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[16:23]
-; GFX1250-NEXT: ; asyncmark(ALL)
-; GFX1250-NEXT: ; wait_asyncmark(0, ALL)
+; GFX1250-NEXT: ; asyncmark(stages=all)
+; GFX1250-NEXT: ; wait_asyncmark(0, stages=all)
; GFX1250-NEXT: s_wait_asynccnt 0x0
; GFX1250-NEXT: s_wait_tensorcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
- call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
+ call void @llvm.amdgcn.wait.asyncmark(i16 0, i32 0)
ret void
}
@@ -455,7 +455,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.1: ; %g1
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-SDAG-NEXT: s_mov_b32 s0, 0
-; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: .LBB14_2: ; %Flow1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
@@ -464,7 +464,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-SDAG-NEXT: ; %bb.3: ; %t1
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: .LBB14_4: ; %merge1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
@@ -472,7 +472,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.5: ; %g2
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-SDAG-NEXT: s_mov_b32 s0, 0
-; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: .LBB14_6: ; %Flow
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
@@ -482,9 +482,9 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-SDAG-NEXT: ; %bb.7: ; %t2
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0xa
; GFX1250-SDAG-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-SDAG-NEXT: ; asyncmark(ALL)
+; GFX1250-SDAG-NEXT: ; asyncmark(stages=all)
; GFX1250-SDAG-NEXT: .LBB14_8: ; %merge2
-; GFX1250-SDAG-NEXT: ; wait_asyncmark(1, ALL)
+; GFX1250-SDAG-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-SDAG-NEXT: s_wait_asynccnt 0x0
; GFX1250-SDAG-NEXT: s_wait_tensorcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -511,7 +511,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.1: ; %g1
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: .LBB14_2: ; %Flow1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_xor_b32 s0, s0, 1
@@ -519,7 +519,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB14_4
; GFX1250-GISEL-NEXT: ; %bb.3: ; %t1
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: .LBB14_4: ; %merge1
; GFX1250-GISEL-NEXT: s_cmp_eq_u32 s1, 0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 1
@@ -527,7 +527,7 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.5: ; %g2
; GFX1250-GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: .LBB14_6: ; %Flow
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_xor_b32 s0, s0, 1
@@ -536,9 +536,9 @@ define void @tensor_or_async_lds_diamonds(i32 inreg %cond1, i32 inreg %cond2, <4
; GFX1250-GISEL-NEXT: ; %bb.7: ; %t2
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0xa
; GFX1250-GISEL-NEXT: tensor_load_to_lds s[12:15], s[4:11]
-; GFX1250-GISEL-NEXT: ; asyncmark(ALL)
+; GFX1250-GISEL-NEXT: ; asyncmark(stages=all)
; GFX1250-GISEL-NEXT: .LBB14_8: ; %merge2
-; GFX1250-GISEL-NEXT: ; wait_asyncmark(1, ALL)
+; GFX1250-GISEL-NEXT: ; wait_asyncmark(1, stages=all)
; GFX1250-GISEL-NEXT: s_wait_asynccnt 0x0
; GFX1250-GISEL-NEXT: s_wait_tensorcnt 0x0
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -548,12 +548,12 @@ entry:
t1:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge1
g1:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge1
merge1:
@@ -562,15 +562,15 @@ merge1:
t2:
call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer, <8 x i32> zeroinitializer, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge2
g2:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, ptr addrspace(3) %dst, i32 0, i32 0)
- call void @llvm.amdgcn.asyncmark(i32 16)
+ call void @llvm.amdgcn.asyncmark(i32 0)
br label %merge2
merge2:
- call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 16)
+ call void @llvm.amdgcn.wait.asyncmark(i16 1, i32 0)
ret void
}
diff --git a/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir b/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
new file mode 100644
index 0000000000000..85d72a2c0d032
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/asyncmark-stage-err.mir
@@ -0,0 +1,29 @@
+# RUN: split-file %s %t
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none %t/mark.mir -filetype=null 2>&1 | FileCheck %s -check-prefix=MARK
+# RUN: not --crash llc -mtriple=amdgpu12.50 -run-pass=none %t/wait.mir -filetype=null 2>&1 | FileCheck %s -check-prefix=WAIT
+
+#--- mark.mir
+# MARK: *** Bad machine code: invalid asyncmark stage mask ***
+# MARK-NEXT: - function: asyncmark_mask_out_of_range
+# MARK: ASYNCMARK 2048
+---
+name: asyncmark_mask_out_of_range
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ASYNCMARK 2048
+ S_ENDPGM 0
+...
+
+#--- wait.mir
+# WAIT: *** Bad machine code: invalid asyncmark stage mask ***
+# WAIT-NEXT: - function: wait_asyncmark_mask_out_of_range
+# WAIT: WAIT_ASYNCMARK 0, 4096
+---
+name: wait_asyncmark_mask_out_of_range
+tracksRegLiveness: true
+body: |
+ bb.0:
+ WAIT_ASYNCMARK 0, 4096
+ S_ENDPGM 0
+...
More information about the cfe-commits
mailing list